Feature/deployment k8s ci cd

.github/workflows/build-release.yml

@@ -0,0 +1,127 @@
+name: Build and Release
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+    tags:
+      - "v*"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  packages: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  validate-helm:
+    name: Validate Helm chart
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up Helm
+        uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4
+
+      - name: Lint chart
+        run: helm lint deploy/helm/newsletter-maker
+
+      - name: Render chart
+        run: helm template newsletter-maker deploy/helm/newsletter-maker -f
+          deploy/helm/newsletter-maker/values-minikube.yaml >
+          /tmp/newsletter-maker-chart.yaml
+
+  build-frontend:
+    name: Build frontend
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: "22"
+          cache: npm
+          cache-dependency-path: frontend/package-lock.json
+
+      - name: Install frontend dependencies
+        working-directory: frontend
+        run: npm ci
+
+      - name: Prepare frontend env
+        working-directory: frontend
+        run: |
+          cp .env.example .env.local
+          echo "NEXTAUTH_SECRET=ci-build-secret" >> .env.local
+
+      - name: Build frontend
+        working-directory: frontend
+        env:
+          NEXT_PUBLIC_API_URL: http://localhost:8000
+          NEXTAUTH_URL: http://localhost:3000
+          NEXTAUTH_SECRET: ci-build-secret
+        run: npm run build
+
+  build-backend:
+    name: Build and scan backend image
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Build backend image
+        env:
+          DOCKER_BUILDKIT: "1"
+        run: docker build -t newsletter-maker-ci:${{ github.sha }} -f
+          docker/web/Dockerfile .
+
+      - name: Scan backend image with Trivy
+        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
+        with:
+          image-ref: newsletter-maker-ci:${{ github.sha }}
+          scan-type: image
+          severity: HIGH,CRITICAL
+          ignore-unfixed: true
+          exit-code: "1"
+
+      - name: Log in to GHCR
+        if: github.event_name == 'push'
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Publish backend image
+        if: github.event_name == 'push'
+        env:
+          IMAGE_REPOSITORY: ghcr.io/${{ github.repository_owner }}/newsletter-maker
+        run: |
+          set -euo pipefail
+
+          docker tag newsletter-maker-ci:${GITHUB_SHA} ${IMAGE_REPOSITORY}:${GITHUB_SHA}
+          docker push ${IMAGE_REPOSITORY}:${GITHUB_SHA}
+
+          if [[ "${GITHUB_REF}" == "refs/heads/main" ]]; then
+            docker tag newsletter-maker-ci:${GITHUB_SHA} ${IMAGE_REPOSITORY}:main
+            docker push ${IMAGE_REPOSITORY}:main
+          fi
+
+          if [[ "${GITHUB_REF}" == refs/tags/* ]]; then
+            version_tag="${GITHUB_REF#refs/tags/}"
+            docker tag newsletter-maker-ci:${GITHUB_SHA} ${IMAGE_REPOSITORY}:${version_tag}
+            docker push ${IMAGE_REPOSITORY}:${version_tag}
+            docker tag newsletter-maker-ci:${GITHUB_SHA} ${IMAGE_REPOSITORY}:latest
+            docker push ${IMAGE_REPOSITORY}:latest
+          fi

.github/workflows/lint.yml

@@ -36,16 +36,19 @@ jobs:
           cache-dependency-path: frontend/package-lock.json
 
       - name: Install just
-        uses: extractions/setup-crate@v2
+        uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2
         with:
           repo: casey/just@1.50.0
           github-token: ${{ github.token }}
 
+      - name: Set up Helm
+        uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4
+
       - name: Install dependencies
         run: just install
 
       - name: Install pre-commit hooks
         run: pre-commit install --install-hooks
 
-      - name: Run lint
+      - name: Run lint and type checks
         run: just lint

.github/workflows/test.yml

@@ -36,7 +36,7 @@ jobs:
           cache-dependency-path: frontend/package-lock.json
 
       - name: Install just
-        uses: extractions/setup-crate@v2
+        uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2
         with:
           repo: casey/just@1.50.0
           github-token: ${{ github.token }}

.pre-commit-config.yaml

@@ -3,6 +3,7 @@ repos:
     rev: v5.0.0
     hooks:
       - id: check-yaml
+        exclude: ^deploy/helm/.+/templates/.*\.(ya?ml)$
       - id: end-of-file-fixer
       - id: trailing-whitespace
 

.vscode/settings.json

@@ -46,6 +46,7 @@
     "upserted",
     "upvote",
     "uritemplate",
+    "xrpc",
     "xxhash"
   ]
 }

README.md

@@ -81,18 +81,10 @@ The system is designed for graceful failure, not silent corruption. Unparseable
 
 **Deployment:** Docker Compose (MVP) · Kubernetes-ready · 12-factor configuration
 
-## Implementation Plan
-
-| Phase | Focus | Key Deliverables |
-| ----- | ----- | ---------------- |
-| **1. MVP** | Content ingestion + basic surfacing | RSS and Reddit plugins · Entity model in Postgres · Qdrant for embeddings · Classification, relevance scoring, and summarization skills · Dashboard with upvote/downvote · Seed script for demo data |
-| **2. Authority** | Newsletter ingestion + authority signals | Resend email intake with LLM extraction · Subscription confirmation flow · Authority scoring from mention frequency · Bluesky plugin · Deduplication and entity extraction skills |
-| **3. Intelligence** | Expanded sources + trend analysis | Mastodon plugin · Trend velocity detection · Theme suggestions · Source diversity analysis · Original content idea generation |
-| **4. Polish** | Advanced features | LinkedIn integration · Automated entity discovery · Full multi-signal authority model · Newsletter draft generation |
-
 ## Project Documentation
 
 - [Developer Guide](docs/DEVELOPER_GUIDE.md) gives a fast "where to look first" map for new contributors.
+- [Deployment Guide](docs/DEPLOYMENT.md) covers Docker Compose, Helm, Minikube, and deployment-aware CI.
 - [Implementation Overview](docs/IMPLEMENTATION_OVERVIEW.md) summarizes the main features and current architecture.
 - [Data Models](docs/MODELS.md) describes the purpose of each core model.
 - [Relevance Scoring](docs/RELEVANCE_SCORING.md) explains how similarity scoring and review thresholds work.

deploy/helm/newsletter-maker/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: newsletter-maker
+description: Helm chart for the Newsletter Maker backend stack.
+type: application
+version: 0.1.0
+appVersion: "0.1.0"

deploy/helm/newsletter-maker/templates/NOTES.txt

@@ -0,0 +1,8 @@
+1. Install the chart into minikube:
+   helm upgrade --install newsletter-maker ./deploy/helm/newsletter-maker -f ./deploy/helm/newsletter-maker/values-minikube.yaml
+
+2. Check the rollout state:
+   kubectl get pods -l app.kubernetes.io/instance={{ .Release.Name }}
+
+3. Reach the cluster service:
+   kubectl port-forward svc/{{ include "newsletter-maker.fullname" . }}-nginx 8080:{{ .Values.nginx.service.port }}

deploy/helm/newsletter-maker/templates/_helpers.tpl

@@ -0,0 +1,56 @@
+{{- define "newsletter-maker.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "newsletter-maker.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name (include "newsletter-maker.name" .) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "newsletter-maker.labels" -}}
+app.kubernetes.io/name: {{ include "newsletter-maker.name" . }}
+helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+{{- define "newsletter-maker.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "newsletter-maker.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{- define "newsletter-maker.componentLabels" -}}
+{{ include "newsletter-maker.selectorLabels" . }}
+app.kubernetes.io/component: {{ .component }}
+{{- end -}}
+
+{{- define "newsletter-maker.databaseHost" -}}
+{{- printf "%s-postgres" (include "newsletter-maker.fullname" .) -}}
+{{- end -}}
+
+{{- define "newsletter-maker.redisHost" -}}
+{{- printf "%s-redis" (include "newsletter-maker.fullname" .) -}}
+{{- end -}}
+
+{{- define "newsletter-maker.qdrantHost" -}}
+{{- printf "%s-qdrant" (include "newsletter-maker.fullname" .) -}}
+{{- end -}}
+
+{{- define "newsletter-maker.djangoHost" -}}
+{{- printf "%s-django" (include "newsletter-maker.fullname" .) -}}
+{{- end -}}
+
+{{- define "newsletter-maker.databaseUrl" -}}
+{{- printf "postgresql://%s:%s@%s:%v/%s" .Values.postgres.username .Values.postgres.password (include "newsletter-maker.databaseHost" .) .Values.postgres.service.port .Values.postgres.database -}}
+{{- end -}}
+
+{{- define "newsletter-maker.redisUrl" -}}
+{{- printf "redis://%s:%v/0" (include "newsletter-maker.redisHost" .) .Values.redis.service.port -}}
+{{- end -}}
+
+{{- define "newsletter-maker.qdrantUrl" -}}
+{{- printf "http://%s:%v" (include "newsletter-maker.qdrantHost" .) .Values.qdrant.service.port -}}
+{{- end -}}

deploy/helm/newsletter-maker/templates/celery-beat-deployment.yaml

@@ -0,0 +1,32 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "newsletter-maker.fullname" . }}-celery-beat
+  labels:
+    {{- include "newsletter-maker.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.celeryBeat.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "newsletter-maker.componentLabels" (dict "Release" .Release "Values" .Values "Chart" .Chart "component" "celery-beat") | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "newsletter-maker.componentLabels" (dict "Release" .Release "Values" .Values "Chart" .Chart "component" "celery-beat") | nindent 8 }}
+    spec:
+      containers:
+        - name: celery-beat
+          image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command:
+            {{- toYaml .Values.celeryBeat.command | nindent 12 }}
+          env:
+            - name: BOOTSTRAP_APP
+              value: "false"
+          envFrom:
+            - configMapRef:
+                name: {{ include "newsletter-maker.fullname" . }}-env
+            - secretRef:
+                name: {{ include "newsletter-maker.fullname" . }}-secret
+          resources:
+            {{- toYaml .Values.celeryBeat.resources | nindent 12 }}

deploy/helm/newsletter-maker/templates/celery-worker-deployment.yaml

@@ -0,0 +1,32 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "newsletter-maker.fullname" . }}-celery-worker
+  labels:
+    {{- include "newsletter-maker.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.celeryWorker.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "newsletter-maker.componentLabels" (dict "Release" .Release "Values" .Values "Chart" .Chart "component" "celery-worker") | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "newsletter-maker.componentLabels" (dict "Release" .Release "Values" .Values "Chart" .Chart "component" "celery-worker") | nindent 8 }}
+    spec:
+      containers:
+        - name: celery-worker
+          image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command:
+            {{- toYaml .Values.celeryWorker.command | nindent 12 }}
+          env:
+            - name: BOOTSTRAP_APP
+              value: "false"
+          envFrom:
+            - configMapRef:
+                name: {{ include "newsletter-maker.fullname" . }}-env
+            - secretRef:
+                name: {{ include "newsletter-maker.fullname" . }}-secret
+          resources:
+            {{- toYaml .Values.celeryWorker.resources | nindent 12 }}

deploy/helm/newsletter-maker/templates/configmap.yaml

@@ -0,0 +1,38 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "newsletter-maker.fullname" . }}-env
+  labels:
+    {{- include "newsletter-maker.labels" . | nindent 4 }}
+data:
+  DEBUG: {{ .Values.env.debug | quote }}
+  ALLOWED_HOSTS: {{ .Values.env.allowedHosts | quote }}
+  CSRF_TRUSTED_ORIGINS: {{ .Values.env.csrfTrustedOrigins | quote }}
+  SITE_ID: {{ .Values.env.siteId | quote }}
+  REDIS_URL: {{ include "newsletter-maker.redisUrl" . | quote }}
+  QDRANT_URL: {{ include "newsletter-maker.qdrantUrl" . | quote }}
+  NEWSLETTER_API_BASE_URL: {{ .Values.env.newsletterApiBaseUrl | quote }}
+  EMAIL_BACKEND: {{ .Values.env.emailBackend | quote }}
+  DEFAULT_FROM_EMAIL: {{ .Values.env.defaultFromEmail | quote }}
+  SERVER_EMAIL: {{ .Values.env.serverEmail | quote }}
+  RESEND_FROM_EMAIL: {{ .Values.env.resendFromEmail | quote }}
+  REDDIT_USER_AGENT: {{ .Values.env.redditUserAgent | quote }}
+  LOG_LEVEL: {{ .Values.env.logLevel | quote }}
+  CELERY_TASK_ALWAYS_EAGER: {{ .Values.env.celeryTaskAlwaysEager | quote }}
+  OPENROUTER_API_BASE: {{ .Values.env.openrouterApiBase | quote }}
+  OPENROUTER_APP_URL: {{ .Values.env.openrouterAppUrl | quote }}
+  OPENROUTER_APP_NAME: {{ .Values.env.openrouterAppName | quote }}
+  AI_CLASSIFICATION_MODEL: {{ .Values.env.aiClassificationModel | quote }}
+  AI_RELEVANCE_MODEL: {{ .Values.env.aiRelevanceModel | quote }}
+  AI_SUMMARIZATION_MODEL: {{ .Values.env.aiSummarizationModel | quote }}
+  AI_CLASSIFICATION_REVIEW_THRESHOLD: {{ .Values.env.aiClassificationReviewThreshold | quote }}
+  AI_RELEVANCE_LOW_THRESHOLD: {{ .Values.env.aiRelevanceLowThreshold | quote }}
+  AI_RELEVANCE_HIGH_THRESHOLD: {{ .Values.env.aiRelevanceHighThreshold | quote }}
+  AI_RELEVANCE_REVIEW_THRESHOLD: {{ .Values.env.aiRelevanceReviewThreshold | quote }}
+  AI_RELEVANCE_SUMMARIZE_THRESHOLD: {{ .Values.env.aiRelevanceSummarizeThreshold | quote }}
+  AI_MAX_NODE_RETRIES: {{ .Values.env.aiMaxNodeRetries | quote }}
+  AI_REQUEST_TIMEOUT_SECONDS: {{ .Values.env.aiRequestTimeoutSeconds | quote }}
+  EMBEDDING_PROVIDER: {{ .Values.env.embeddingProvider | quote }}
+  EMBEDDING_MODEL: {{ .Values.env.embeddingModel | quote }}
+  EMBEDDING_TRUST_REMOTE_CODE: {{ .Values.env.embeddingTrustRemoteCode | quote }}
+  OLLAMA_URL: {{ .Values.env.ollamaUrl | quote }}