fix: remove dead metrics RBAC, document NetworkPolicy prereq, add scrape-down alert

config/monitoring/prometheus-rule.yaml

@@ -80,3 +80,13 @@ spec:
           annotations:
             summary: "SeiNodeGroup reconcile substep {{ $labels.substep }} is slow"
             description: "p99 latency above 10s for 10 minutes."
+
+        - alert: ControllerMetricsDown
+          expr: up{job=~".*sei-k8s-controller.*"} == 0
+          for: 5m
+          labels:
+            severity: critical
+            team: platform
+          annotations:
+            summary: "sei-k8s-controller metrics endpoint is down"
+            description: "Prometheus has been unable to scrape the controller for 5 minutes. All other controller alerts are blind."

config/monitoring/service-monitor.yaml

@@ -8,6 +8,7 @@ metadata:
     app.kubernetes.io/name: sei-k8s-controller
     app.kubernetes.io/managed-by: kustomize
 spec:
+  jobLabel: app.kubernetes.io/name
   selector:
     matchLabels:
       control-plane: controller-manager

config/network-policy/allow-metrics-traffic.yaml

@@ -1,3 +1,7 @@
+# Allows Prometheus to scrape the controller metrics endpoint.
+# Prerequisite: the Prometheus namespace must carry the label
+#   kubectl label namespace <prometheus-ns> metrics=enabled
+# Without this label, scrapes will be blocked by the namespaceSelector.
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:

config/rbac/kustomization.yaml

@@ -6,6 +6,3 @@ resources:
   - role_binding.yaml
   - leader_election_role.yaml
   - leader_election_role_binding.yaml
-  - metrics_auth_role.yaml
-  - metrics_auth_role_binding.yaml
-  - metrics_reader_role.yaml