ROX-28976: Optimize berserker load in long running cluster (#64)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

Author: JoukoVirtanen

release/start-kube-burner/start-kube-burner.sh

@@ -39,8 +39,6 @@ temp_metrics_file="${DIR}"/metrics.yml
 sed '/captureStart/d' "${KUBE_BURNER_METRICS_FILE}" > "$temp_metrics_file"
 kubectl create configmap --from-file="$temp_metrics_file" kube-burner-metrics-config -n kube-burner
 
-kubectl create configmap --from-file="$KUBE_BURNER_METRICS_FILE" kube-burner-metrics-config -n kube-burner
-
 kubectl create -f "${DIR}"/service-account.yaml
 kubectl create -f "${DIR}"/cluster-role-binding.yaml
 

release/start-secured-cluster/prometheus.yaml

@@ -0,0 +1,84 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus
+  namespace: stackrox
+  labels:
+    app.kubernetes.io/name: stackrox
+data:
+  prometheus.yml: |-
+    global:
+      scrape_interval: 30s
+
+    alerting:
+      alertmanagers:
+        - static_configs:
+            - targets:
+                - stackrox-monitoring-alertmanager:9093
+
+    rule_files:
+      - /etc/prometheus/rules_*.yml
+
+    scrape_configs:
+      - job_name: "kubernetes-pods"
+        tls_config:
+          insecure_skip_verify: false
+        kubernetes_sd_configs:
+          - role: pod
+            namespaces:
+              own_namespace: true
+        relabel_configs:
+          - action: labelmap
+            regex: __meta_kubernetes_pod_label_(.+)
+          - source_labels: [__meta_kubernetes_namespace]
+            action: replace
+            target_label: namespace
+          - source_labels: [__meta_kubernetes_pod_name]
+            action: replace
+            target_label: pod
+          - source_labels: [__meta_kubernetes_pod_node_name]
+            action: replace
+            target_label: node_name
+
+      - job_name: "kubernetes-cadvisor"
+        scheme: https
+        metrics_path: /metrics/cadvisor
+        tls_config:
+          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+          insecure_skip_verify: true
+        authorization:
+          credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+        kubernetes_sd_configs:
+          - role: node
+        relabel_configs:
+          - action: labelmap
+            regex: __meta_kubernetes_node_label_(.+)
+
+        metric_relabel_configs:
+          - source_labels: [container]
+            regex: berserker
+            action: drop
+          - source_labels: [namespace]
+            regex: berserker-*
+            action: drop
+
+      - job_name: stackrox
+        tls_config:
+          insecure_skip_verify: false
+        kubernetes_sd_configs:
+          - role: endpoints
+            namespaces:
+              own_namespace: true
+        relabel_configs:
+          - source_labels: [__meta_kubernetes_endpoint_port_name]
+            action: keep
+            regex: monitoring
+          - source_labels: [__meta_kubernetes_endpoints_name]
+            action: replace
+            target_label: job
+          - source_labels: [__meta_kubernetes_namespace]
+            action: replace
+            target_label: namespace
+          - source_labels: [__meta_kubernetes_endpoint_node_name]
+            action: replace
+            target_label: node_name

release/start-secured-cluster/start-secured-cluster.sh

@@ -13,13 +13,20 @@ kubectl -n stackrox create secret generic access-rhacs \
 kubectl create -f "${SCRIPT_DIR}/collector-config.yaml"
 
 echo "Deploying Monitoring..."
+monitoring_values_file="${COMMON_DIR}/../charts/monitoring/values.yaml"
+yq -i '.resources.requests.memory = "8Gi"' "$monitoring_values_file"
+yq -i '.resources.limits.memory = "8Gi"' "$monitoring_values_file"
+
 helm_args=(
   --set persistence.type="${STORAGE}"
   --set exposure.type="${MONITORING_LOAD_BALANCER}"
 )
 
 helm dependency update "${COMMON_DIR}/../charts/monitoring"
-envsubst < "${COMMON_DIR}/../charts/monitoring/values.yaml" > "${COMMON_DIR}/../charts/monitoring/values_substituted.yaml"
+envsubst < "$monitoring_values_file" > "${COMMON_DIR}/../charts/monitoring/values_substituted.yaml"
 helm upgrade -n stackrox --install --create-namespace stackrox-monitoring "${COMMON_DIR}/../charts/monitoring" --values "${COMMON_DIR}/../charts/monitoring/values_substituted.yaml" "${helm_args[@]}"
 rm "${COMMON_DIR}/../charts/monitoring/values_substituted.yaml"
-echo "Deployed Monitoring..."
+
+# Replace the prometheus ConfigMap with one that doesn't scrape as much info from berserker containers
+kubectl -n stackrox delete configmap prometheus
+kubectl create -f "${SCRIPT_DIR}"/prometheus.yaml