From 12162dfe3bd9569f0717ae4ef670b6bceb02301e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcelo=20Gon=C3=A7alves?= Date: Wed, 6 May 2026 13:25:21 +0100 Subject: [PATCH 1/6] WIP --- chart/templates/external-secret.yaml | 20 ++++++++++++++++++++ chart/values.yaml | 8 +++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/chart/templates/external-secret.yaml b/chart/templates/external-secret.yaml index ea39748..6e3ad2d 100644 --- a/chart/templates/external-secret.yaml +++ b/chart/templates/external-secret.yaml @@ -3,10 +3,21 @@ apiVersion: external-secrets.io/v1 kind: SecretStore metadata: + {{- if eq ((.Values.externalSecrets).provider | default "aws") "azure-keyvault" }} + name: azure-keyvault + {{- else }} name: aws-secretsmanager + {{- end }} namespace: {{ .Release.Namespace }} spec: provider: + {{- if eq ((.Values.externalSecrets).provider | default "aws") "azure-keyvault" }} + azurekv: + authType: WorkloadIdentity + vaultUrl: {{ .Values.externalSecrets.azureKeyVault.vaultUrl }} + serviceAccountRef: + name: {{ .Values.externalSecrets.serviceAccount.name | default "external-secrets-sa" }} + {{- else }} aws: service: SecretsManager region: {{ .Values.global.awsRegion | default "eu-central-1" }} @@ -14,6 +25,7 @@ spec: jwt: serviceAccountRef: name: {{ .Values.externalSecrets.serviceAccount.name | default "external-secrets-sa" }} + {{- end }} --- {{- if .Values.externalSecrets.serviceAccount.create }} apiVersion: v1 @@ -25,6 +37,10 @@ metadata: annotations: {{- toYaml . | nindent 4 }} {{- end }} + {{- with .Values.externalSecrets.serviceAccount.labels }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} --- apiVersion: external-secrets.io/v1 @@ -35,7 +51,11 @@ metadata: spec: refreshInterval: 1h secretStoreRef: + {{- if eq ((.Values.externalSecrets).provider | default "aws") "azure-keyvault" }} + name: azure-keyvault + {{- else }} name: aws-secretsmanager + {{- end }} kind: SecretStore target: name: {{ .Values.secretEnv.existingSecret | default "openops-env" }} diff --git a/chart/values.yaml b/chart/values.yaml index 0e0a3b4..57dfc41 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -631,14 +631,20 @@ serviceMonitor: # Example: release: prometheus # External Secrets Operator configuration (optional) -# When enabled, secrets are synced from AWS Secrets Manager, GCP Secret Manager, etc. +# When enabled, secrets are synced from AWS Secrets Manager, Azure Key Vault, GCP Secret Manager, etc. externalSecrets: enabled: false + # Provider: "aws" (default) or "azure-keyvault" + provider: "aws" secretName: "" + # Azure Key Vault configuration (only used when provider is "azure-keyvault") + azureKeyVault: + vaultUrl: "" # e.g., "https://my-keyvault.vault.azure.net/" serviceAccount: create: false name: external-secrets-sa annotations: {} + labels: {} # Subagent configuration subagents: From fa2cbc694c98f32a9b1a51a541d1dbd8007d21bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcelo=20Gon=C3=A7alves?= Date: Fri, 8 May 2026 13:32:57 +0100 Subject: [PATCH 2/6] WIP --- README.md | 76 +++++++- chart/templates/_helpers.tpl | 22 ++- chart/templates/configmap-analytics.yaml | 11 ++ .../cronjob-ecr-credential-refresh.yaml | 164 ++++++++++++++++++ chart/templates/deployment-analytics.yaml | 10 ++ chart/templates/deployment-app.yaml | 3 +- chart/templates/deployment-engine.yaml | 2 +- chart/templates/deployment-nginx.yaml | 16 +- chart/templates/deployment-tables.yaml | 1 + chart/templates/external-secret.yaml | 1 + chart/values.yaml | 24 +++ 11 files changed, 325 insertions(+), 5 deletions(-) create mode 100644 chart/templates/configmap-analytics.yaml create mode 100644 chart/templates/cronjob-ecr-credential-refresh.yaml diff --git a/README.md b/README.md index e5b4fc9..b8784aa 100644 --- a/README.md +++ b/README.md @@ -315,6 +315,38 @@ kubectl create secret tls openops-tls \ ## Dependencies The deployments include health checks and readiness probes so dependent services wait until their prerequisites are available. +## Private ECR access from non-AWS clusters + +When deploying on AKS or other non-AWS Kubernetes clusters that need to pull images from a private ECR registry, enable the ECR credential refresh CronJob: + +```yaml +global: + imagePullSecrets: + - name: ecr-pull-secret + +ecrCredentialRefresh: + enabled: true + registry: "" + awsRegion: "us-east-2" + awsSecretName: "ecr-credentials" # K8s secret with AWS access keys + imagePullSecretName: "ecr-pull-secret" # Created/refreshed automatically +``` + +**Pre-requisite:** Create the AWS credentials secret (one-time): +```bash +kubectl create secret generic ecr-credentials \ + -n openops \ + --from-literal=aws-access-key-id= \ + --from-literal=aws-secret-access-key= +``` + +The IAM user needs only `ecr:GetAuthorizationToken`, `ecr:BatchGetImage`, and `ecr:GetDownloadUrlForLayer` permissions. + +**How it works:** +- A Helm post-install/post-upgrade hook creates the pull secret immediately on deploy +- A CronJob refreshes the ECR token every 6 hours (tokens expire after 12h) +- All deployments reference the pull secret via `global.imagePullSecrets` + ## Topology and rollout safeguards The chart provides built-in safeguards to avoid single-node concentration and ensure safe rolling updates: @@ -478,7 +510,49 @@ secretEnv: Create secrets using one of these methods: -**External Secrets Operator:** +**External Secrets Operator (AWS Secrets Manager):** + +The chart has built-in support for External Secrets with AWS Secrets Manager. Enable it in your values: +```yaml +externalSecrets: + enabled: true + provider: "aws" # default + secretName: "my-aws-secret" # Name in AWS Secrets Manager + serviceAccount: + create: true + name: external-secrets-sa + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::123456789:role/external-secrets-role + +secretEnv: + create: false + existingSecret: openops-env +``` + +**External Secrets Operator (Azure Key Vault):** + +For Azure AKS deployments using Workload Identity: +```yaml +externalSecrets: + enabled: true + provider: "azure-keyvault" + azureKeyVault: + vaultUrl: "https://my-keyvault.vault.azure.net/" + secretName: "my-keyvault-secret" + serviceAccount: + create: true + name: external-secrets-sa + annotations: + azure.workload.identity/client-id: "" + labels: + azure.workload.identity/use: "true" + +secretEnv: + create: false + existingSecret: openops-env +``` + +**External Secrets Operator (manual):** ```yaml apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl index cba5689..250a640 100644 --- a/chart/templates/_helpers.tpl +++ b/chart/templates/_helpers.tpl @@ -221,12 +221,16 @@ Expected dict: { "root": $, "env": dict } {{- define "openops.renderEnv" -}} {{- $root := .root -}} {{- $env := .env -}} +{{- $skipDuplicateSecrets := .skipDuplicateSecrets | default false -}} +{{- $secrets := default (dict) $root.Values.openopsEnvSecrets -}} {{- if $env }} {{- range $k, $v := $env }} +{{- if or (not $skipDuplicateSecrets) (not (hasKey $secrets $k)) }} {{ include "openops.envVar" (dict "root" $root "key" $k "value" $v) }} {{- end }} {{- end }} {{- end }} +{{- end }} {{/* Resolve the AWS Secrets Manager property name for a secret key. @@ -254,12 +258,28 @@ Expected dict: { "root": $, "env": dict, "secretName": "my-secret" } {{- $root := .root -}} {{- $env := .env -}} {{- $secretName := .secretName -}} +{{- $infraSecretName := "" -}} +{{- $appSecretName := "" -}} +{{- if $root.Values.externalSecrets.infraSecretName -}} + {{- $infraSecretName = $root.Values.externalSecrets.infraSecretName -}} + {{- $appSecretName = $root.Values.externalSecrets.appSecretName -}} +{{- end -}} +{{- $infraKeys := list "OPS_POSTGRES_PASSWORD" "OPS_REDIS_URL" "OPS_REDIS_HOST" "OPS_REDIS_PASSWORD" -}} {{- range $k := keys $env | sortAlpha -}} {{- $v := index $env $k -}} {{- if eq (include "openops.isSecretKey" (dict "root" $root "key" $k "value" ($v | toString))) "true" }} +{{- $remoteSecret := $secretName -}} +{{- if $infraSecretName -}} + {{- $propName := include "openops.secretPropertyName" (dict "key" $k "value" ($v | toString)) -}} + {{- if has $propName $infraKeys -}} + {{- $remoteSecret = $infraSecretName -}} + {{- else -}} + {{- $remoteSecret = $appSecretName -}} + {{- end -}} +{{- end }} - secretKey: {{ $k }} remoteRef: - key: {{ $secretName }} + key: {{ $remoteSecret }} property: {{ include "openops.secretPropertyName" (dict "key" $k "value" ($v | toString)) }} {{- end -}} {{- end -}} diff --git a/chart/templates/configmap-analytics.yaml b/chart/templates/configmap-analytics.yaml new file mode 100644 index 0000000..b263740 --- /dev/null +++ b/chart/templates/configmap-analytics.yaml @@ -0,0 +1,11 @@ +{{- if .Values.analytics.configOverride }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.analytics.name }}-config + labels: + {{- include "openops.componentLabels" (dict "root" . "component" "analytics") | nindent 4 }} +data: + superset_config_docker.py: | +{{ .Values.analytics.configOverride | indent 4 }} +{{- end }} diff --git a/chart/templates/cronjob-ecr-credential-refresh.yaml b/chart/templates/cronjob-ecr-credential-refresh.yaml new file mode 100644 index 0000000..9b272e6 --- /dev/null +++ b/chart/templates/cronjob-ecr-credential-refresh.yaml @@ -0,0 +1,164 @@ +{{- if .Values.ecrCredentialRefresh.enabled }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.ecrCredentialRefresh.serviceAccount.name | default "ecr-credential-refresh" }} + namespace: {{ .Release.Namespace }} + {{- with .Values.ecrCredentialRefresh.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ecr-credential-refresh + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ecr-credential-refresh + namespace: {{ .Release.Namespace }} +subjects: + - kind: ServiceAccount + name: {{ .Values.ecrCredentialRefresh.serviceAccount.name | default "ecr-credential-refresh" }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: ecr-credential-refresh + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: ecr-credential-refresh + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: ecr-credential-refresh + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + schedule: "0 */6 * * *" + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 3 + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 3 + template: + spec: + serviceAccountName: {{ .Values.ecrCredentialRefresh.serviceAccount.name | default "ecr-credential-refresh" }} + restartPolicy: OnFailure + containers: + - name: ecr-credential-refresh + image: amazon/aws-cli:2.27.18 + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.ecrCredentialRefresh.awsSecretName | default "ecr-credentials" }} + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.ecrCredentialRefresh.awsSecretName | default "ecr-credentials" }} + key: AWS_SECRET_ACCESS_KEY + - name: AWS_REGION + value: {{ .Values.ecrCredentialRefresh.awsRegion | quote }} + - name: ECR_REGISTRY + value: {{ .Values.ecrCredentialRefresh.registry | quote }} + - name: SECRET_NAME + value: {{ .Values.ecrCredentialRefresh.imagePullSecretName | default "ecr-pull-secret" | quote }} + - name: NAMESPACE + value: {{ .Release.Namespace }} + command: + - /bin/bash + - -c + - | + set -euo pipefail + curl -sLO "https://dl.k8s.io/release/$(curl -sL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x kubectl && mv kubectl /usr/local/bin/ + TOKEN=$(aws ecr get-login-password --region "$AWS_REGION") + kubectl delete secret "$SECRET_NAME" -n "$NAMESPACE" --ignore-not-found + kubectl create secret docker-registry "$SECRET_NAME" \ + -n "$NAMESPACE" \ + --docker-server="$ECR_REGISTRY" \ + --docker-username=AWS \ + --docker-password="$TOKEN" + echo "✅ ECR pull secret refreshed successfully" + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "128Mi" + cpu: "100m" +--- +# Initial Job to create the secret immediately on install/upgrade +apiVersion: batch/v1 +kind: Job +metadata: + name: ecr-credential-refresh-init + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app.kubernetes.io/name: ecr-credential-refresh + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + backoffLimit: 3 + template: + spec: + serviceAccountName: {{ .Values.ecrCredentialRefresh.serviceAccount.name | default "ecr-credential-refresh" }} + restartPolicy: OnFailure + containers: + - name: ecr-credential-refresh + image: amazon/aws-cli:2.27.18 + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.ecrCredentialRefresh.awsSecretName | default "ecr-credentials" }} + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.ecrCredentialRefresh.awsSecretName | default "ecr-credentials" }} + key: AWS_SECRET_ACCESS_KEY + - name: AWS_REGION + value: {{ .Values.ecrCredentialRefresh.awsRegion | quote }} + - name: ECR_REGISTRY + value: {{ .Values.ecrCredentialRefresh.registry | quote }} + - name: SECRET_NAME + value: {{ .Values.ecrCredentialRefresh.imagePullSecretName | default "ecr-pull-secret" | quote }} + - name: NAMESPACE + value: {{ .Release.Namespace }} + command: + - /bin/bash + - -c + - | + set -euo pipefail + curl -sLO "https://dl.k8s.io/release/$(curl -sL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x kubectl && mv kubectl /usr/local/bin/ + TOKEN=$(aws ecr get-login-password --region "$AWS_REGION") + kubectl delete secret "$SECRET_NAME" -n "$NAMESPACE" --ignore-not-found + kubectl create secret docker-registry "$SECRET_NAME" \ + -n "$NAMESPACE" \ + --docker-server="$ECR_REGISTRY" \ + --docker-username=AWS \ + --docker-password="$TOKEN" + echo "✅ ECR pull secret created successfully" + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "128Mi" + cpu: "100m" +{{- end }} diff --git a/chart/templates/deployment-analytics.yaml b/chart/templates/deployment-analytics.yaml index e18b5d9..d4128a5 100644 --- a/chart/templates/deployment-analytics.yaml +++ b/chart/templates/deployment-analytics.yaml @@ -77,3 +77,13 @@ spec: timeoutSeconds: 8 failureThreshold: 10 {{- include "openops.lifecyclePreStop" . | nindent 8 }} +{{- if .Values.analytics.configOverride }} + volumeMounts: + - name: config-override + mountPath: /app/pythonpath/superset_config_docker.py + subPath: superset_config_docker.py + volumes: + - name: config-override + configMap: + name: {{ .Values.analytics.name }}-config +{{- end }} diff --git a/chart/templates/deployment-app.yaml b/chart/templates/deployment-app.yaml index df0e275..b19bb48 100644 --- a/chart/templates/deployment-app.yaml +++ b/chart/templates/deployment-app.yaml @@ -67,13 +67,14 @@ spec: - FOWNER - SETUID - SETGID + - NET_BIND_SERVICE {{- end }} env: - name: NODE_OPTIONS value: "--no-node-snapshot" - name: OPS_COMPONENT value: app -{{ include "openops.renderEnv" (dict "root" . "env" .Values.openopsEnv) | nindent 12 }} +{{ include "openops.renderEnv" (dict "root" . "env" .Values.openopsEnv "skipDuplicateSecrets" true) | nindent 12 }} {{ include "openops.renderEnv" (dict "root" . "env" .Values.openopsEnvSecrets) | nindent 12 }} ports: - containerPort: 80 diff --git a/chart/templates/deployment-engine.yaml b/chart/templates/deployment-engine.yaml index 14a87e1..6b567c6 100644 --- a/chart/templates/deployment-engine.yaml +++ b/chart/templates/deployment-engine.yaml @@ -56,7 +56,7 @@ spec: value: "--no-node-snapshot" - name: OPS_COMPONENT value: engine -{{ include "openops.renderEnv" (dict "root" . "env" .Values.openopsEnv) | nindent 12 }} +{{ include "openops.renderEnv" (dict "root" . "env" .Values.openopsEnv "skipDuplicateSecrets" true) | nindent 12 }} {{ include "openops.renderEnv" (dict "root" . "env" .Values.openopsEnvSecrets) | nindent 12 }} {{ include "openops.renderEnv" (dict "root" . "env" .Values.engine.env) | nindent 12 }} ports: diff --git a/chart/templates/deployment-nginx.yaml b/chart/templates/deployment-nginx.yaml index 31643dd..c10bfae 100644 --- a/chart/templates/deployment-nginx.yaml +++ b/chart/templates/deployment-nginx.yaml @@ -47,7 +47,21 @@ spec: - name: {{ .Values.nginx.name }} image: {{ .Values.nginx.image }}:{{ .Values.nginx.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- include "openops.containerSecurityContext" (dict "root" .) | nindent 8 }} + {{- if .Values.global.containerSecurityContext.enabled }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + runAsNonRoot: false + runAsUser: 0 + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + - CHOWN + - SETGID + - SETUID + {{- end }} ports: - containerPort: 80 name: http diff --git a/chart/templates/deployment-tables.yaml b/chart/templates/deployment-tables.yaml index 52e573d..3e540d3 100644 --- a/chart/templates/deployment-tables.yaml +++ b/chart/templates/deployment-tables.yaml @@ -66,6 +66,7 @@ spec: - FOWNER - SETUID - SETGID + - NET_BIND_SERVICE {{- end }} env: {{ include "openops.renderEnv" (dict "root" . "env" .Values.tables.env) | nindent 10 }} diff --git a/chart/templates/external-secret.yaml b/chart/templates/external-secret.yaml index 6e3ad2d..a19e6b1 100644 --- a/chart/templates/external-secret.yaml +++ b/chart/templates/external-secret.yaml @@ -15,6 +15,7 @@ spec: azurekv: authType: WorkloadIdentity vaultUrl: {{ .Values.externalSecrets.azureKeyVault.vaultUrl }} + tenantId: {{ .Values.externalSecrets.azureKeyVault.tenantId }} serviceAccountRef: name: {{ .Values.externalSecrets.serviceAccount.name | default "external-secrets-sa" }} {{- else }} diff --git a/chart/values.yaml b/chart/values.yaml index 57dfc41..e59e254 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -367,8 +367,13 @@ analytics: SUPERSET_FEATURE_ALLOW_ADHOC_SUBQUERY: '"{{ .Values.openopsEnv.ANALYTICS_ALLOW_ADHOC_SUBQUERY }}"' REDIS_HOST: '{{ include "openops.redisUrlExtractHost" . }}' REDIS_PORT: '{{ include "openops.redisUrlExtractPort" . }}' + REDIS_USERNAME: '' + REDIS_PASSWORD: '' + REDIS_SSL: 'false' REDIS_CELERY_DB: '{{ include "openops.redisUrlExtractDb" . }}' REDIS_RESULTS_DB: '{{ include "openops.redisUrlExtractDb" . }}' + # Override superset_config_docker.py (mounted into the container) + configOverride: "" postgres: name: postgres @@ -640,12 +645,31 @@ externalSecrets: # Azure Key Vault configuration (only used when provider is "azure-keyvault") azureKeyVault: vaultUrl: "" # e.g., "https://my-keyvault.vault.azure.net/" + tenantId: "" # Azure AD tenant ID + infraSecretName: "" # Terraform-managed Key Vault secret (auto-updated) + appSecretName: "" # Manually-managed Key Vault secret serviceAccount: create: false name: external-secrets-sa annotations: {} labels: {} +# ECR credential refresh CronJob (for pulling from private ECR outside AWS) +# Creates and refreshes an imagePullSecret every 6 hours using AWS ECR token. +# Required when AKS or non-AWS clusters need to pull from private ECR. +ecrCredentialRefresh: + enabled: false + registry: "" + # AWS region of the ECR registry + awsRegion: "us-east-2" + # Name of the Kubernetes secret containing AWS credentials (keys: aws-access-key-id, aws-secret-access-key) + awsSecretName: "ecr-credentials" + # Name of the imagePullSecret that will be created/refreshed + imagePullSecretName: "ecr-pull-secret" + serviceAccount: + name: "ecr-credential-refresh" + annotations: {} + # Subagent configuration subagents: # Namespace where subagent pods run (defaults to same namespace as app) From 6d19312ee0f728b62ea837e1bbbc71f6b235a21c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcelo=20Gon=C3=A7alves?= Date: Fri, 8 May 2026 13:47:00 +0100 Subject: [PATCH 3/6] fix: address PR review comments - Add required validation for vaultUrl/tenantId when azure-keyvault provider - Add fail for unsupported externalSecrets.provider values - Extract $provider variable to reduce duplication in external-secret.yaml - Restrict ECR cronjob RBAC to specific secret resourceNames - Pin kubectl version to v1.32.3 in ECR cronjob - Fix appSecretName fallback when empty (use secretName instead) - Update renderEnv comment to document skipDuplicateSecrets param - Fix analytics configOverride whitespace (avoid left-trim concatenation) - Fix values.yaml comment for ECR secret key names (AWS_ACCESS_KEY_ID) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- chart/templates/_helpers.tpl | 5 +++-- .../templates/cronjob-ecr-credential-refresh.yaml | 12 +++++++++--- chart/templates/deployment-analytics.yaml | 2 +- chart/templates/external-secret.yaml | 15 +++++++++------ chart/values.yaml | 2 +- 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl index 250a640..abf6c4b 100644 --- a/chart/templates/_helpers.tpl +++ b/chart/templates/_helpers.tpl @@ -216,7 +216,8 @@ Expected dict: { "root": $, "key": "ENV", "value": "value" } {{/* Render environment variables from a map using openops.envVar. -Expected dict: { "root": $, "env": dict } +Expected dict: { "root": $, "env": dict, "skipDuplicateSecrets": bool (optional) } +When skipDuplicateSecrets is true, keys already present in openopsEnvSecrets are skipped. */}} {{- define "openops.renderEnv" -}} {{- $root := .root -}} @@ -273,7 +274,7 @@ Expected dict: { "root": $, "env": dict, "secretName": "my-secret" } {{- $propName := include "openops.secretPropertyName" (dict "key" $k "value" ($v | toString)) -}} {{- if has $propName $infraKeys -}} {{- $remoteSecret = $infraSecretName -}} - {{- else -}} + {{- else if $appSecretName -}} {{- $remoteSecret = $appSecretName -}} {{- end -}} {{- end }} diff --git a/chart/templates/cronjob-ecr-credential-refresh.yaml b/chart/templates/cronjob-ecr-credential-refresh.yaml index 9b272e6..08cb9ac 100644 --- a/chart/templates/cronjob-ecr-credential-refresh.yaml +++ b/chart/templates/cronjob-ecr-credential-refresh.yaml @@ -18,7 +18,11 @@ metadata: rules: - apiGroups: [""] resources: ["secrets"] - verbs: ["get", "create", "patch", "delete"] + resourceNames: [{{ .Values.ecrCredentialRefresh.imagePullSecretName | default "ecr-pull-secret" | quote }}] + verbs: ["get", "patch", "delete"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -81,7 +85,8 @@ spec: - -c - | set -euo pipefail - curl -sLO "https://dl.k8s.io/release/$(curl -sL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + KUBECTL_VERSION="v1.32.3" + curl -sLO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" chmod +x kubectl && mv kubectl /usr/local/bin/ TOKEN=$(aws ecr get-login-password --region "$AWS_REGION") kubectl delete secret "$SECRET_NAME" -n "$NAMESPACE" --ignore-not-found @@ -144,7 +149,8 @@ spec: - -c - | set -euo pipefail - curl -sLO "https://dl.k8s.io/release/$(curl -sL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + KUBECTL_VERSION="v1.32.3" + curl -sLO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" chmod +x kubectl && mv kubectl /usr/local/bin/ TOKEN=$(aws ecr get-login-password --region "$AWS_REGION") kubectl delete secret "$SECRET_NAME" -n "$NAMESPACE" --ignore-not-found diff --git a/chart/templates/deployment-analytics.yaml b/chart/templates/deployment-analytics.yaml index d4128a5..cf06973 100644 --- a/chart/templates/deployment-analytics.yaml +++ b/chart/templates/deployment-analytics.yaml @@ -77,7 +77,7 @@ spec: timeoutSeconds: 8 failureThreshold: 10 {{- include "openops.lifecyclePreStop" . | nindent 8 }} -{{- if .Values.analytics.configOverride }} +{{ if .Values.analytics.configOverride }} volumeMounts: - name: config-override mountPath: /app/pythonpath/superset_config_docker.py diff --git a/chart/templates/external-secret.yaml b/chart/templates/external-secret.yaml index a19e6b1..6d74d71 100644 --- a/chart/templates/external-secret.yaml +++ b/chart/templates/external-secret.yaml @@ -3,19 +3,22 @@ apiVersion: external-secrets.io/v1 kind: SecretStore metadata: - {{- if eq ((.Values.externalSecrets).provider | default "aws") "azure-keyvault" }} + {{- $provider := (.Values.externalSecrets).provider | default "aws" }} + {{- if eq $provider "azure-keyvault" }} name: azure-keyvault - {{- else }} + {{- else if eq $provider "aws" }} name: aws-secretsmanager + {{- else }} + {{- fail (printf "Unsupported externalSecrets.provider: %s. Supported values: aws, azure-keyvault" $provider) }} {{- end }} namespace: {{ .Release.Namespace }} spec: provider: - {{- if eq ((.Values.externalSecrets).provider | default "aws") "azure-keyvault" }} + {{- if eq $provider "azure-keyvault" }} azurekv: authType: WorkloadIdentity - vaultUrl: {{ .Values.externalSecrets.azureKeyVault.vaultUrl }} - tenantId: {{ .Values.externalSecrets.azureKeyVault.tenantId }} + vaultUrl: {{ required "externalSecrets.azureKeyVault.vaultUrl is required when provider is azure-keyvault" .Values.externalSecrets.azureKeyVault.vaultUrl }} + tenantId: {{ required "externalSecrets.azureKeyVault.tenantId is required when provider is azure-keyvault" .Values.externalSecrets.azureKeyVault.tenantId }} serviceAccountRef: name: {{ .Values.externalSecrets.serviceAccount.name | default "external-secrets-sa" }} {{- else }} @@ -52,7 +55,7 @@ metadata: spec: refreshInterval: 1h secretStoreRef: - {{- if eq ((.Values.externalSecrets).provider | default "aws") "azure-keyvault" }} + {{- if eq $provider "azure-keyvault" }} name: azure-keyvault {{- else }} name: aws-secretsmanager diff --git a/chart/values.yaml b/chart/values.yaml index e59e254..5ec1ae6 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -662,7 +662,7 @@ ecrCredentialRefresh: registry: "" # AWS region of the ECR registry awsRegion: "us-east-2" - # Name of the Kubernetes secret containing AWS credentials (keys: aws-access-key-id, aws-secret-access-key) + # Name of the Kubernetes secret containing AWS credentials (keys: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) awsSecretName: "ecr-credentials" # Name of the imagePullSecret that will be created/refreshed imagePullSecretName: "ecr-pull-secret" From 3d6fe6d9f355f11347bae6ae16f5d8371a0654f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcelo=20Gon=C3=A7alves?= Date: Fri, 8 May 2026 13:50:46 +0100 Subject: [PATCH 4/6] docs: update README with tenantId, correct ECR secret keys, and analytics configOverride Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b8784aa..66c6854 100644 --- a/README.md +++ b/README.md @@ -336,8 +336,8 @@ ecrCredentialRefresh: ```bash kubectl create secret generic ecr-credentials \ -n openops \ - --from-literal=aws-access-key-id= \ - --from-literal=aws-secret-access-key= + --from-literal=AWS_ACCESS_KEY_ID= \ + --from-literal=AWS_SECRET_ACCESS_KEY= ``` The IAM user needs only `ecr:GetAuthorizationToken`, `ecr:BatchGetImage`, and `ecr:GetDownloadUrlForLayer` permissions. @@ -347,6 +347,48 @@ The IAM user needs only `ecr:GetAuthorizationToken`, `ecr:BatchGetImage`, and `e - A CronJob refreshes the ECR token every 6 hours (tokens expire after 12h) - All deployments reference the pull secret via `global.imagePullSecrets` +## Analytics (Superset) configuration override + +The analytics component is based on Apache Superset. To override its Python configuration (e.g., for Redis SSL/auth), use `analytics.configOverride`: + +```yaml +analytics: + env: + REDIS_HOST: "my-redis.example.com" + REDIS_PORT: "6380" + REDIS_PASSWORD: "my-password" + REDIS_SSL: "true" + configOverride: | + import os + + REDIS_HOST = os.getenv("REDIS_HOST", "redis") + REDIS_PORT = os.getenv("REDIS_PORT", "6379") + REDIS_PASSWORD = os.getenv("REDIS_PASSWORD", "") + REDIS_SSL = os.getenv("REDIS_SSL", "false").lower() == "true" + + REDIS_SCHEME = "rediss" if REDIS_SSL else "redis" + REDIS_AUTH = f"default:{REDIS_PASSWORD}@" if REDIS_PASSWORD else "" + REDIS_BASE_URL = f"{REDIS_SCHEME}://{REDIS_AUTH}{REDIS_HOST}:{REDIS_PORT}" + + CACHE_CONFIG = { + "CACHE_TYPE": "RedisCache", + "CACHE_DEFAULT_TIMEOUT": 300, + "CACHE_KEY_PREFIX": "superset_", + "CACHE_REDIS_URL": f"{REDIS_BASE_URL}/0", + } + DATA_CACHE_CONFIG = CACHE_CONFIG + + class CeleryConfig: + broker_url = f"{REDIS_BASE_URL}/0" + result_backend = f"{REDIS_BASE_URL}/0" + worker_prefetch_multiplier = 1 + task_acks_late = False + + CELERY_CONFIG = CeleryConfig +``` + +This mounts the config as `/app/pythonpath/superset_config_docker.py` inside the container, which is imported by the base Superset config and overrides cache/Celery settings. + ## Topology and rollout safeguards The chart provides built-in safeguards to avoid single-node concentration and ensure safe rolling updates: @@ -538,7 +580,10 @@ externalSecrets: provider: "azure-keyvault" azureKeyVault: vaultUrl: "https://my-keyvault.vault.azure.net/" + tenantId: "" secretName: "my-keyvault-secret" + infraSecretName: "infra-secrets" # Optional: Terraform-managed secret + appSecretName: "app-secrets" # Optional: Manually-managed secret serviceAccount: create: true name: external-secrets-sa From aebd6ed8bb9aa93f8544ea2e229cc906069beea5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcelo=20Gon=C3=A7alves?= Date: Fri, 8 May 2026 14:41:00 +0100 Subject: [PATCH 5/6] =?UTF-8?q?fix:=20address=20review=20comments=20?= =?UTF-8?q?=E2=80=94=20checksum,=20registry=20validation,=20idempotent=20s?= =?UTF-8?q?ecret=20refresh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - README: fix imagePullSecrets example to use string list (not objects) - deployment-analytics: add checksum/analytics-config annotation for rollout on configOverride changes - cronjob-ecr-credential-refresh: require registry when enabled via Helm required() - cronjob-ecr-credential-refresh: use dry-run + apply instead of delete + create to avoid secret gap Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 2 +- chart/templates/cronjob-ecr-credential-refresh.yaml | 12 ++++++------ chart/templates/deployment-analytics.yaml | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 66c6854..3dc1650 100644 --- a/README.md +++ b/README.md @@ -322,7 +322,7 @@ When deploying on AKS or other non-AWS Kubernetes clusters that need to pull ima ```yaml global: imagePullSecrets: - - name: ecr-pull-secret + - ecr-pull-secret ecrCredentialRefresh: enabled: true diff --git a/chart/templates/cronjob-ecr-credential-refresh.yaml b/chart/templates/cronjob-ecr-credential-refresh.yaml index 08cb9ac..66836a8 100644 --- a/chart/templates/cronjob-ecr-credential-refresh.yaml +++ b/chart/templates/cronjob-ecr-credential-refresh.yaml @@ -75,7 +75,7 @@ spec: - name: AWS_REGION value: {{ .Values.ecrCredentialRefresh.awsRegion | quote }} - name: ECR_REGISTRY - value: {{ .Values.ecrCredentialRefresh.registry | quote }} + value: {{ required "ecrCredentialRefresh.registry is required when ecrCredentialRefresh is enabled" .Values.ecrCredentialRefresh.registry | quote }} - name: SECRET_NAME value: {{ .Values.ecrCredentialRefresh.imagePullSecretName | default "ecr-pull-secret" | quote }} - name: NAMESPACE @@ -89,12 +89,12 @@ spec: curl -sLO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" chmod +x kubectl && mv kubectl /usr/local/bin/ TOKEN=$(aws ecr get-login-password --region "$AWS_REGION") - kubectl delete secret "$SECRET_NAME" -n "$NAMESPACE" --ignore-not-found kubectl create secret docker-registry "$SECRET_NAME" \ -n "$NAMESPACE" \ --docker-server="$ECR_REGISTRY" \ --docker-username=AWS \ - --docker-password="$TOKEN" + --docker-password="$TOKEN" \ + --dry-run=client -o yaml | kubectl apply -f - echo "✅ ECR pull secret refreshed successfully" resources: requests: @@ -139,7 +139,7 @@ spec: - name: AWS_REGION value: {{ .Values.ecrCredentialRefresh.awsRegion | quote }} - name: ECR_REGISTRY - value: {{ .Values.ecrCredentialRefresh.registry | quote }} + value: {{ required "ecrCredentialRefresh.registry is required when ecrCredentialRefresh is enabled" .Values.ecrCredentialRefresh.registry | quote }} - name: SECRET_NAME value: {{ .Values.ecrCredentialRefresh.imagePullSecretName | default "ecr-pull-secret" | quote }} - name: NAMESPACE @@ -153,12 +153,12 @@ spec: curl -sLO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" chmod +x kubectl && mv kubectl /usr/local/bin/ TOKEN=$(aws ecr get-login-password --region "$AWS_REGION") - kubectl delete secret "$SECRET_NAME" -n "$NAMESPACE" --ignore-not-found kubectl create secret docker-registry "$SECRET_NAME" \ -n "$NAMESPACE" \ --docker-server="$ECR_REGISTRY" \ --docker-username=AWS \ - --docker-password="$TOKEN" + --docker-password="$TOKEN" \ + --dry-run=client -o yaml | kubectl apply -f - echo "✅ ECR pull secret created successfully" resources: requests: diff --git a/chart/templates/deployment-analytics.yaml b/chart/templates/deployment-analytics.yaml index cf06973..22dc286 100644 --- a/chart/templates/deployment-analytics.yaml +++ b/chart/templates/deployment-analytics.yaml @@ -31,6 +31,9 @@ spec: {{- with include "openops.secretChecksum" . }} checksum/secret-env: {{ . }} {{- end }} + {{- if .Values.analytics.configOverride }} + checksum/analytics-config: {{ include (print $.Template.BasePath "/configmap-analytics.yaml") . | sha256sum }} + {{- end }} {{- with .Values.global.commonAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} From ec404ee191e92d7128d9c5b2821cb4567e60aa3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcelo=20Gon=C3=A7alves?= Date: Fri, 8 May 2026 15:01:11 +0100 Subject: [PATCH 6/6] fix: add SHA256 verification for kubectl download in ECR credential refresh Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- chart/templates/cronjob-ecr-credential-refresh.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/chart/templates/cronjob-ecr-credential-refresh.yaml b/chart/templates/cronjob-ecr-credential-refresh.yaml index 66836a8..d2d6ec2 100644 --- a/chart/templates/cronjob-ecr-credential-refresh.yaml +++ b/chart/templates/cronjob-ecr-credential-refresh.yaml @@ -86,7 +86,9 @@ spec: - | set -euo pipefail KUBECTL_VERSION="v1.32.3" + KUBECTL_SHA256="ab209d0c5134b61486a0486585604a616a5bb2fc07df46d304b3c95817b2d79f" curl -sLO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + echo "${KUBECTL_SHA256} kubectl" | sha256sum -c - chmod +x kubectl && mv kubectl /usr/local/bin/ TOKEN=$(aws ecr get-login-password --region "$AWS_REGION") kubectl create secret docker-registry "$SECRET_NAME" \ @@ -150,7 +152,9 @@ spec: - | set -euo pipefail KUBECTL_VERSION="v1.32.3" + KUBECTL_SHA256="ab209d0c5134b61486a0486585604a616a5bb2fc07df46d304b3c95817b2d79f" curl -sLO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + echo "${KUBECTL_SHA256} kubectl" | sha256sum -c - chmod +x kubectl && mv kubectl /usr/local/bin/ TOKEN=$(aws ecr get-login-password --region "$AWS_REGION") kubectl create secret docker-registry "$SECRET_NAME" \