diff --git a/README.md b/README.md index e5b4fc9..3dc1650 100644 --- a/README.md +++ b/README.md @@ -315,6 +315,80 @@ kubectl create secret tls openops-tls \ ## Dependencies The deployments include health checks and readiness probes so dependent services wait until their prerequisites are available. +## Private ECR access from non-AWS clusters + +When deploying on AKS or other non-AWS Kubernetes clusters that need to pull images from a private ECR registry, enable the ECR credential refresh CronJob: + +```yaml +global: + imagePullSecrets: + - ecr-pull-secret + +ecrCredentialRefresh: + enabled: true + registry: "" + awsRegion: "us-east-2" + awsSecretName: "ecr-credentials" # K8s secret with AWS access keys + imagePullSecretName: "ecr-pull-secret" # Created/refreshed automatically +``` + +**Pre-requisite:** Create the AWS credentials secret (one-time): +```bash +kubectl create secret generic ecr-credentials \ + -n openops \ + --from-literal=AWS_ACCESS_KEY_ID= \ + --from-literal=AWS_SECRET_ACCESS_KEY= +``` + +The IAM user needs only `ecr:GetAuthorizationToken`, `ecr:BatchGetImage`, and `ecr:GetDownloadUrlForLayer` permissions. + +**How it works:** +- A Helm post-install/post-upgrade hook creates the pull secret immediately on deploy +- A CronJob refreshes the ECR token every 6 hours (tokens expire after 12h) +- All deployments reference the pull secret via `global.imagePullSecrets` + +## Analytics (Superset) configuration override + +The analytics component is based on Apache Superset. To override its Python configuration (e.g., for Redis SSL/auth), use `analytics.configOverride`: + +```yaml +analytics: + env: + REDIS_HOST: "my-redis.example.com" + REDIS_PORT: "6380" + REDIS_PASSWORD: "my-password" + REDIS_SSL: "true" + configOverride: | + import os + + REDIS_HOST = os.getenv("REDIS_HOST", "redis") + REDIS_PORT = os.getenv("REDIS_PORT", "6379") + REDIS_PASSWORD = os.getenv("REDIS_PASSWORD", "") + REDIS_SSL = os.getenv("REDIS_SSL", "false").lower() == "true" + + REDIS_SCHEME = "rediss" if REDIS_SSL else "redis" + REDIS_AUTH = f"default:{REDIS_PASSWORD}@" if REDIS_PASSWORD else "" + REDIS_BASE_URL = f"{REDIS_SCHEME}://{REDIS_AUTH}{REDIS_HOST}:{REDIS_PORT}" + + CACHE_CONFIG = { + "CACHE_TYPE": "RedisCache", + "CACHE_DEFAULT_TIMEOUT": 300, + "CACHE_KEY_PREFIX": "superset_", + "CACHE_REDIS_URL": f"{REDIS_BASE_URL}/0", + } + DATA_CACHE_CONFIG = CACHE_CONFIG + + class CeleryConfig: + broker_url = f"{REDIS_BASE_URL}/0" + result_backend = f"{REDIS_BASE_URL}/0" + worker_prefetch_multiplier = 1 + task_acks_late = False + + CELERY_CONFIG = CeleryConfig +``` + +This mounts the config as `/app/pythonpath/superset_config_docker.py` inside the container, which is imported by the base Superset config and overrides cache/Celery settings. + ## Topology and rollout safeguards The chart provides built-in safeguards to avoid single-node concentration and ensure safe rolling updates: @@ -478,7 +552,52 @@ secretEnv: Create secrets using one of these methods: -**External Secrets Operator:** +**External Secrets Operator (AWS Secrets Manager):** + +The chart has built-in support for External Secrets with AWS Secrets Manager. Enable it in your values: +```yaml +externalSecrets: + enabled: true + provider: "aws" # default + secretName: "my-aws-secret" # Name in AWS Secrets Manager + serviceAccount: + create: true + name: external-secrets-sa + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::123456789:role/external-secrets-role + +secretEnv: + create: false + existingSecret: openops-env +``` + +**External Secrets Operator (Azure Key Vault):** + +For Azure AKS deployments using Workload Identity: +```yaml +externalSecrets: + enabled: true + provider: "azure-keyvault" + azureKeyVault: + vaultUrl: "https://my-keyvault.vault.azure.net/" + tenantId: "" + secretName: "my-keyvault-secret" + infraSecretName: "infra-secrets" # Optional: Terraform-managed secret + appSecretName: "app-secrets" # Optional: Manually-managed secret + serviceAccount: + create: true + name: external-secrets-sa + annotations: + azure.workload.identity/client-id: "" + labels: + azure.workload.identity/use: "true" + +secretEnv: + create: false + existingSecret: openops-env +``` + +**External Secrets Operator (manual):** ```yaml apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl index cba5689..abf6c4b 100644 --- a/chart/templates/_helpers.tpl +++ b/chart/templates/_helpers.tpl @@ -216,17 +216,22 @@ Expected dict: { "root": $, "key": "ENV", "value": "value" } {{/* Render environment variables from a map using openops.envVar. -Expected dict: { "root": $, "env": dict } +Expected dict: { "root": $, "env": dict, "skipDuplicateSecrets": bool (optional) } +When skipDuplicateSecrets is true, keys already present in openopsEnvSecrets are skipped. */}} {{- define "openops.renderEnv" -}} {{- $root := .root -}} {{- $env := .env -}} +{{- $skipDuplicateSecrets := .skipDuplicateSecrets | default false -}} +{{- $secrets := default (dict) $root.Values.openopsEnvSecrets -}} {{- if $env }} {{- range $k, $v := $env }} +{{- if or (not $skipDuplicateSecrets) (not (hasKey $secrets $k)) }} {{ include "openops.envVar" (dict "root" $root "key" $k "value" $v) }} {{- end }} {{- end }} {{- end }} +{{- end }} {{/* Resolve the AWS Secrets Manager property name for a secret key. @@ -254,12 +259,28 @@ Expected dict: { "root": $, "env": dict, "secretName": "my-secret" } {{- $root := .root -}} {{- $env := .env -}} {{- $secretName := .secretName -}} +{{- $infraSecretName := "" -}} +{{- $appSecretName := "" -}} +{{- if $root.Values.externalSecrets.infraSecretName -}} + {{- $infraSecretName = $root.Values.externalSecrets.infraSecretName -}} + {{- $appSecretName = $root.Values.externalSecrets.appSecretName -}} +{{- end -}} +{{- $infraKeys := list "OPS_POSTGRES_PASSWORD" "OPS_REDIS_URL" "OPS_REDIS_HOST" "OPS_REDIS_PASSWORD" -}} {{- range $k := keys $env | sortAlpha -}} {{- $v := index $env $k -}} {{- if eq (include "openops.isSecretKey" (dict "root" $root "key" $k "value" ($v | toString))) "true" }} +{{- $remoteSecret := $secretName -}} +{{- if $infraSecretName -}} + {{- $propName := include "openops.secretPropertyName" (dict "key" $k "value" ($v | toString)) -}} + {{- if has $propName $infraKeys -}} + {{- $remoteSecret = $infraSecretName -}} + {{- else if $appSecretName -}} + {{- $remoteSecret = $appSecretName -}} + {{- end -}} +{{- end }} - secretKey: {{ $k }} remoteRef: - key: {{ $secretName }} + key: {{ $remoteSecret }} property: {{ include "openops.secretPropertyName" (dict "key" $k "value" ($v | toString)) }} {{- end -}} {{- end -}} diff --git a/chart/templates/configmap-analytics.yaml b/chart/templates/configmap-analytics.yaml new file mode 100644 index 0000000..b263740 --- /dev/null +++ b/chart/templates/configmap-analytics.yaml @@ -0,0 +1,11 @@ +{{- if .Values.analytics.configOverride }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.analytics.name }}-config + labels: + {{- include "openops.componentLabels" (dict "root" . "component" "analytics") | nindent 4 }} +data: + superset_config_docker.py: | +{{ .Values.analytics.configOverride | indent 4 }} +{{- end }} diff --git a/chart/templates/cronjob-ecr-credential-refresh.yaml b/chart/templates/cronjob-ecr-credential-refresh.yaml new file mode 100644 index 0000000..d2d6ec2 --- /dev/null +++ b/chart/templates/cronjob-ecr-credential-refresh.yaml @@ -0,0 +1,174 @@ +{{- if .Values.ecrCredentialRefresh.enabled }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.ecrCredentialRefresh.serviceAccount.name | default "ecr-credential-refresh" }} + namespace: {{ .Release.Namespace }} + {{- with .Values.ecrCredentialRefresh.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ecr-credential-refresh + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: [""] + resources: ["secrets"] + resourceNames: [{{ .Values.ecrCredentialRefresh.imagePullSecretName | default "ecr-pull-secret" | quote }}] + verbs: ["get", "patch", "delete"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ecr-credential-refresh + namespace: {{ .Release.Namespace }} +subjects: + - kind: ServiceAccount + name: {{ .Values.ecrCredentialRefresh.serviceAccount.name | default "ecr-credential-refresh" }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: ecr-credential-refresh + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: ecr-credential-refresh + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: ecr-credential-refresh + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + schedule: "0 */6 * * *" + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 3 + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 3 + template: + spec: + serviceAccountName: {{ .Values.ecrCredentialRefresh.serviceAccount.name | default "ecr-credential-refresh" }} + restartPolicy: OnFailure + containers: + - name: ecr-credential-refresh + image: amazon/aws-cli:2.27.18 + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.ecrCredentialRefresh.awsSecretName | default "ecr-credentials" }} + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.ecrCredentialRefresh.awsSecretName | default "ecr-credentials" }} + key: AWS_SECRET_ACCESS_KEY + - name: AWS_REGION + value: {{ .Values.ecrCredentialRefresh.awsRegion | quote }} + - name: ECR_REGISTRY + value: {{ required "ecrCredentialRefresh.registry is required when ecrCredentialRefresh is enabled" .Values.ecrCredentialRefresh.registry | quote }} + - name: SECRET_NAME + value: {{ .Values.ecrCredentialRefresh.imagePullSecretName | default "ecr-pull-secret" | quote }} + - name: NAMESPACE + value: {{ .Release.Namespace }} + command: + - /bin/bash + - -c + - | + set -euo pipefail + KUBECTL_VERSION="v1.32.3" + KUBECTL_SHA256="ab209d0c5134b61486a0486585604a616a5bb2fc07df46d304b3c95817b2d79f" + curl -sLO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + echo "${KUBECTL_SHA256} kubectl" | sha256sum -c - + chmod +x kubectl && mv kubectl /usr/local/bin/ + TOKEN=$(aws ecr get-login-password --region "$AWS_REGION") + kubectl create secret docker-registry "$SECRET_NAME" \ + -n "$NAMESPACE" \ + --docker-server="$ECR_REGISTRY" \ + --docker-username=AWS \ + --docker-password="$TOKEN" \ + --dry-run=client -o yaml | kubectl apply -f - + echo "✅ ECR pull secret refreshed successfully" + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "128Mi" + cpu: "100m" +--- +# Initial Job to create the secret immediately on install/upgrade +apiVersion: batch/v1 +kind: Job +metadata: + name: ecr-credential-refresh-init + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app.kubernetes.io/name: ecr-credential-refresh + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + backoffLimit: 3 + template: + spec: + serviceAccountName: {{ .Values.ecrCredentialRefresh.serviceAccount.name | default "ecr-credential-refresh" }} + restartPolicy: OnFailure + containers: + - name: ecr-credential-refresh + image: amazon/aws-cli:2.27.18 + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.ecrCredentialRefresh.awsSecretName | default "ecr-credentials" }} + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.ecrCredentialRefresh.awsSecretName | default "ecr-credentials" }} + key: AWS_SECRET_ACCESS_KEY + - name: AWS_REGION + value: {{ .Values.ecrCredentialRefresh.awsRegion | quote }} + - name: ECR_REGISTRY + value: {{ required "ecrCredentialRefresh.registry is required when ecrCredentialRefresh is enabled" .Values.ecrCredentialRefresh.registry | quote }} + - name: SECRET_NAME + value: {{ .Values.ecrCredentialRefresh.imagePullSecretName | default "ecr-pull-secret" | quote }} + - name: NAMESPACE + value: {{ .Release.Namespace }} + command: + - /bin/bash + - -c + - | + set -euo pipefail + KUBECTL_VERSION="v1.32.3" + KUBECTL_SHA256="ab209d0c5134b61486a0486585604a616a5bb2fc07df46d304b3c95817b2d79f" + curl -sLO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + echo "${KUBECTL_SHA256} kubectl" | sha256sum -c - + chmod +x kubectl && mv kubectl /usr/local/bin/ + TOKEN=$(aws ecr get-login-password --region "$AWS_REGION") + kubectl create secret docker-registry "$SECRET_NAME" \ + -n "$NAMESPACE" \ + --docker-server="$ECR_REGISTRY" \ + --docker-username=AWS \ + --docker-password="$TOKEN" \ + --dry-run=client -o yaml | kubectl apply -f - + echo "✅ ECR pull secret created successfully" + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "128Mi" + cpu: "100m" +{{- end }} diff --git a/chart/templates/deployment-analytics.yaml b/chart/templates/deployment-analytics.yaml index e18b5d9..22dc286 100644 --- a/chart/templates/deployment-analytics.yaml +++ b/chart/templates/deployment-analytics.yaml @@ -31,6 +31,9 @@ spec: {{- with include "openops.secretChecksum" . }} checksum/secret-env: {{ . }} {{- end }} + {{- if .Values.analytics.configOverride }} + checksum/analytics-config: {{ include (print $.Template.BasePath "/configmap-analytics.yaml") . | sha256sum }} + {{- end }} {{- with .Values.global.commonAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} @@ -77,3 +80,13 @@ spec: timeoutSeconds: 8 failureThreshold: 10 {{- include "openops.lifecyclePreStop" . | nindent 8 }} +{{ if .Values.analytics.configOverride }} + volumeMounts: + - name: config-override + mountPath: /app/pythonpath/superset_config_docker.py + subPath: superset_config_docker.py + volumes: + - name: config-override + configMap: + name: {{ .Values.analytics.name }}-config +{{- end }} diff --git a/chart/templates/deployment-app.yaml b/chart/templates/deployment-app.yaml index df0e275..b19bb48 100644 --- a/chart/templates/deployment-app.yaml +++ b/chart/templates/deployment-app.yaml @@ -67,13 +67,14 @@ spec: - FOWNER - SETUID - SETGID + - NET_BIND_SERVICE {{- end }} env: - name: NODE_OPTIONS value: "--no-node-snapshot" - name: OPS_COMPONENT value: app -{{ include "openops.renderEnv" (dict "root" . "env" .Values.openopsEnv) | nindent 12 }} +{{ include "openops.renderEnv" (dict "root" . "env" .Values.openopsEnv "skipDuplicateSecrets" true) | nindent 12 }} {{ include "openops.renderEnv" (dict "root" . "env" .Values.openopsEnvSecrets) | nindent 12 }} ports: - containerPort: 80 diff --git a/chart/templates/deployment-engine.yaml b/chart/templates/deployment-engine.yaml index 14a87e1..6b567c6 100644 --- a/chart/templates/deployment-engine.yaml +++ b/chart/templates/deployment-engine.yaml @@ -56,7 +56,7 @@ spec: value: "--no-node-snapshot" - name: OPS_COMPONENT value: engine -{{ include "openops.renderEnv" (dict "root" . "env" .Values.openopsEnv) | nindent 12 }} +{{ include "openops.renderEnv" (dict "root" . "env" .Values.openopsEnv "skipDuplicateSecrets" true) | nindent 12 }} {{ include "openops.renderEnv" (dict "root" . "env" .Values.openopsEnvSecrets) | nindent 12 }} {{ include "openops.renderEnv" (dict "root" . "env" .Values.engine.env) | nindent 12 }} ports: diff --git a/chart/templates/deployment-nginx.yaml b/chart/templates/deployment-nginx.yaml index 31643dd..c10bfae 100644 --- a/chart/templates/deployment-nginx.yaml +++ b/chart/templates/deployment-nginx.yaml @@ -47,7 +47,21 @@ spec: - name: {{ .Values.nginx.name }} image: {{ .Values.nginx.image }}:{{ .Values.nginx.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- include "openops.containerSecurityContext" (dict "root" .) | nindent 8 }} + {{- if .Values.global.containerSecurityContext.enabled }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + runAsNonRoot: false + runAsUser: 0 + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + - CHOWN + - SETGID + - SETUID + {{- end }} ports: - containerPort: 80 name: http diff --git a/chart/templates/deployment-tables.yaml b/chart/templates/deployment-tables.yaml index 52e573d..3e540d3 100644 --- a/chart/templates/deployment-tables.yaml +++ b/chart/templates/deployment-tables.yaml @@ -66,6 +66,7 @@ spec: - FOWNER - SETUID - SETGID + - NET_BIND_SERVICE {{- end }} env: {{ include "openops.renderEnv" (dict "root" . "env" .Values.tables.env) | nindent 10 }} diff --git a/chart/templates/external-secret.yaml b/chart/templates/external-secret.yaml index ea39748..6d74d71 100644 --- a/chart/templates/external-secret.yaml +++ b/chart/templates/external-secret.yaml @@ -3,10 +3,25 @@ apiVersion: external-secrets.io/v1 kind: SecretStore metadata: + {{- $provider := (.Values.externalSecrets).provider | default "aws" }} + {{- if eq $provider "azure-keyvault" }} + name: azure-keyvault + {{- else if eq $provider "aws" }} name: aws-secretsmanager + {{- else }} + {{- fail (printf "Unsupported externalSecrets.provider: %s. Supported values: aws, azure-keyvault" $provider) }} + {{- end }} namespace: {{ .Release.Namespace }} spec: provider: + {{- if eq $provider "azure-keyvault" }} + azurekv: + authType: WorkloadIdentity + vaultUrl: {{ required "externalSecrets.azureKeyVault.vaultUrl is required when provider is azure-keyvault" .Values.externalSecrets.azureKeyVault.vaultUrl }} + tenantId: {{ required "externalSecrets.azureKeyVault.tenantId is required when provider is azure-keyvault" .Values.externalSecrets.azureKeyVault.tenantId }} + serviceAccountRef: + name: {{ .Values.externalSecrets.serviceAccount.name | default "external-secrets-sa" }} + {{- else }} aws: service: SecretsManager region: {{ .Values.global.awsRegion | default "eu-central-1" }} @@ -14,6 +29,7 @@ spec: jwt: serviceAccountRef: name: {{ .Values.externalSecrets.serviceAccount.name | default "external-secrets-sa" }} + {{- end }} --- {{- if .Values.externalSecrets.serviceAccount.create }} apiVersion: v1 @@ -25,6 +41,10 @@ metadata: annotations: {{- toYaml . | nindent 4 }} {{- end }} + {{- with .Values.externalSecrets.serviceAccount.labels }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} --- apiVersion: external-secrets.io/v1 @@ -35,7 +55,11 @@ metadata: spec: refreshInterval: 1h secretStoreRef: + {{- if eq $provider "azure-keyvault" }} + name: azure-keyvault + {{- else }} name: aws-secretsmanager + {{- end }} kind: SecretStore target: name: {{ .Values.secretEnv.existingSecret | default "openops-env" }} diff --git a/chart/values.yaml b/chart/values.yaml index 0e0a3b4..5ec1ae6 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -367,8 +367,13 @@ analytics: SUPERSET_FEATURE_ALLOW_ADHOC_SUBQUERY: '"{{ .Values.openopsEnv.ANALYTICS_ALLOW_ADHOC_SUBQUERY }}"' REDIS_HOST: '{{ include "openops.redisUrlExtractHost" . }}' REDIS_PORT: '{{ include "openops.redisUrlExtractPort" . }}' + REDIS_USERNAME: '' + REDIS_PASSWORD: '' + REDIS_SSL: 'false' REDIS_CELERY_DB: '{{ include "openops.redisUrlExtractDb" . }}' REDIS_RESULTS_DB: '{{ include "openops.redisUrlExtractDb" . }}' + # Override superset_config_docker.py (mounted into the container) + configOverride: "" postgres: name: postgres @@ -631,14 +636,39 @@ serviceMonitor: # Example: release: prometheus # External Secrets Operator configuration (optional) -# When enabled, secrets are synced from AWS Secrets Manager, GCP Secret Manager, etc. +# When enabled, secrets are synced from AWS Secrets Manager, Azure Key Vault, GCP Secret Manager, etc. externalSecrets: enabled: false + # Provider: "aws" (default) or "azure-keyvault" + provider: "aws" secretName: "" + # Azure Key Vault configuration (only used when provider is "azure-keyvault") + azureKeyVault: + vaultUrl: "" # e.g., "https://my-keyvault.vault.azure.net/" + tenantId: "" # Azure AD tenant ID + infraSecretName: "" # Terraform-managed Key Vault secret (auto-updated) + appSecretName: "" # Manually-managed Key Vault secret serviceAccount: create: false name: external-secrets-sa annotations: {} + labels: {} + +# ECR credential refresh CronJob (for pulling from private ECR outside AWS) +# Creates and refreshes an imagePullSecret every 6 hours using AWS ECR token. +# Required when AKS or non-AWS clusters need to pull from private ECR. +ecrCredentialRefresh: + enabled: false + registry: "" + # AWS region of the ECR registry + awsRegion: "us-east-2" + # Name of the Kubernetes secret containing AWS credentials (keys: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) + awsSecretName: "ecr-credentials" + # Name of the imagePullSecret that will be created/refreshed + imagePullSecretName: "ecr-pull-secret" + serviceAccount: + name: "ecr-credential-refresh" + annotations: {} # Subagent configuration subagents: