From 9a8599f8d1b8d6a47e40ad9e0f11a4a681074277 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Mon, 27 Apr 2026 01:44:19 +0300 Subject: [PATCH 01/17] Add frontend scaffolding --- .env.example | 4 + .gitignore | 2 + core/api.py | 57 +- core/pipeline.py | 128 +- core/tests/test_api.py | 64 + docker-compose.yml | 19 + frontend/.env.example | 5 + frontend/.eslintrc | 0 frontend/.prettierrc | 0 frontend/app/admin/health/page.tsx | 108 + frontend/app/admin/sources/page.tsx | 132 + frontend/app/api/entities/[id]/route.ts | 44 + frontend/app/api/entities/route.ts | 35 + frontend/app/api/feedback/route.ts | 27 + frontend/app/api/review/[id]/route.ts | 29 + frontend/app/api/skills/[skillName]/route.ts | 39 + frontend/app/api/source-configs/[id]/route.ts | 35 + frontend/app/api/source-configs/route.ts | 34 + frontend/app/content/[id]/page.tsx | 197 + frontend/app/entities/page.tsx | 175 + frontend/app/globals.css | 439 ++ frontend/app/layout.tsx | 28 + frontend/app/page.tsx | 298 + frontend/components/app-shell.tsx | 68 + frontend/components/status-badge.tsx | 10 + frontend/eslint.config.mjs | 48 + frontend/lib/api.ts | 157 + frontend/lib/types.ts | 102 + frontend/lib/view-helpers.ts | 66 + frontend/next-env.d.ts | 5 + frontend/next.config.ts | 7 + frontend/package-lock.json | 5826 +++++++++++++++++ frontend/package.json | 36 + frontend/tsconfig.json | 23 + frontend/tsconfig.tsbuildinfo | 1 + justfile | 24 + 36 files changed, 8270 insertions(+), 2 deletions(-) create mode 100644 frontend/.env.example create mode 100644 frontend/.eslintrc create mode 100644 frontend/.prettierrc create mode 100644 frontend/app/admin/health/page.tsx create mode 100644 frontend/app/admin/sources/page.tsx create mode 100644 frontend/app/api/entities/[id]/route.ts create mode 100644 frontend/app/api/entities/route.ts create mode 100644 frontend/app/api/feedback/route.ts create mode 100644 frontend/app/api/review/[id]/route.ts create mode 100644 frontend/app/api/skills/[skillName]/route.ts create mode 100644 frontend/app/api/source-configs/[id]/route.ts create mode 100644 frontend/app/api/source-configs/route.ts create mode 100644 frontend/app/content/[id]/page.tsx create mode 100644 frontend/app/entities/page.tsx create mode 100644 frontend/app/globals.css create mode 100644 frontend/app/layout.tsx create mode 100644 frontend/app/page.tsx create mode 100644 frontend/components/app-shell.tsx create mode 100644 frontend/components/status-badge.tsx create mode 100644 frontend/eslint.config.mjs create mode 100644 frontend/lib/api.ts create mode 100644 frontend/lib/types.ts create mode 100644 frontend/lib/view-helpers.ts create mode 100644 frontend/next-env.d.ts create mode 100644 frontend/next.config.ts create mode 100644 frontend/package-lock.json create mode 100644 frontend/package.json create mode 100644 frontend/tsconfig.json create mode 100644 frontend/tsconfig.tsbuildinfo diff --git a/.env.example b/.env.example index 991d1d85..9c6cfdbe 100644 --- a/.env.example +++ b/.env.example @@ -34,3 +34,7 @@ CELERY_TASK_ALWAYS_EAGER=false DJANGO_SUPERUSER_USERNAME=admin DJANGO_SUPERUSER_EMAIL=admin@example.com DJANGO_SUPERUSER_PASSWORD=adminpass + +NEWSLETTER_API_BASE_URL=http://127.0.0.1:8080 +NEWSLETTER_API_USERNAME=admin +NEWSLETTER_API_PASSWORD=adminpass diff --git a/.gitignore b/.gitignore index 639cdcc9..fd0d17f0 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,7 @@ venv/ celerybeat-schedule* db.sqlite3 staticfiles/ +frontend/.next/ +frontend/node_modules/ docs/ diff --git a/core/api.py b/core/api.py index 46782db9..3f830d16 100644 --- a/core/api.py +++ b/core/api.py @@ -8,8 +8,10 @@ extend_schema_view, inline_serializer, ) -from rest_framework import serializers, viewsets +from rest_framework import serializers, status, viewsets +from rest_framework.decorators import action from rest_framework.exceptions import NotFound +from rest_framework.response import Response from core.models import ( Content, @@ -22,6 +24,13 @@ TenantConfig, UserFeedback, ) +from core.pipeline import ( + CLASSIFICATION_SKILL_NAME, + RELATED_CONTENT_SKILL_NAME, + RELEVANCE_SKILL_NAME, + SUMMARIZATION_SKILL_NAME, + execute_ad_hoc_skill, +) from core.serializers import ( ContentSerializer, EntitySerializer, @@ -41,6 +50,16 @@ description="The unique ID of the tenant that owns this nested resource.", ) +SKILL_NAME_PARAMETER = OpenApiParameter( + name="skill_name", + type=str, + location=OpenApiParameter.PATH, + description=( + "The skill to run for this content item. Supported values: " + "content_classification, relevance_scoring, summarization, find_related." + ), +) + TENANT_CREATE_REQUEST_EXAMPLE = OpenApiExample( "Create Tenant Request", value={ @@ -498,6 +517,42 @@ class ContentViewSet(TenantOwnedQuerysetMixin, viewsets.ModelViewSet): serializer_class = ContentSerializer queryset = Content.objects.select_related("tenant", "entity") + @extend_schema( + summary="Run content skill", + description=( + "Run one ad hoc skill for the selected content item and persist the outcome as a SkillResult. " + "Supported skill names are content_classification, relevance_scoring, summarization, and find_related." + ), + tags=["AI Processing"], + parameters=[TENANT_ID_PARAMETER, SKILL_NAME_PARAMETER], + request=None, + responses={ + 201: SkillResultSerializer, + 403: AUTHENTICATION_REQUIRED_RESPONSE, + }, + ) + @action(detail=True, methods=["post"], url_path=r"skills/(?P[^/.]+)") + def run_skill(self, request, *args, **kwargs): + skill_name = str(kwargs["skill_name"]) + if skill_name not in { + CLASSIFICATION_SKILL_NAME, + RELEVANCE_SKILL_NAME, + SUMMARIZATION_SKILL_NAME, + RELATED_CONTENT_SKILL_NAME, + }: + raise serializers.ValidationError( + { + "skill_name": ( + "Unsupported skill. Choose one of: content_classification, relevance_scoring, " + "summarization, find_related." + ) + } + ) + + skill_result = execute_ad_hoc_skill(self.get_object(), skill_name) + serializer = SkillResultSerializer(skill_result, context=self.get_serializer_context()) + return Response(serializer.data, status=status.HTTP_201_CREATED) + @document_tenant_owned_viewset( resource_plural="skill results", diff --git a/core/pipeline.py b/core/pipeline.py index 521a94db..06569ca4 100644 --- a/core/pipeline.py +++ b/core/pipeline.py @@ -8,7 +8,7 @@ from django.conf import settings from langgraph.graph import END, StateGraph -from core.embeddings import build_content_embedding_text, embed_text, get_reference_similarity +from core.embeddings import build_content_embedding_text, embed_text, get_reference_similarity, search_similar_content from core.llm import openrouter_chat_json from core.models import Content, ReviewQueue, ReviewReason, SkillResult, SkillStatus @@ -17,6 +17,7 @@ CLASSIFICATION_SKILL_NAME = "content_classification" RELEVANCE_SKILL_NAME = "relevance_scoring" SUMMARIZATION_SKILL_NAME = "summarization" +RELATED_CONTENT_SKILL_NAME = "find_related" CONTENT_TYPES = ( "technical_article", @@ -274,6 +275,109 @@ def run_summarization(content: Content) -> dict[str, Any]: } +def execute_ad_hoc_skill(content: Content, skill_name: str) -> SkillResult: + if skill_name == CLASSIFICATION_SKILL_NAME: + return _execute_ad_hoc_classification(content) + if skill_name == RELEVANCE_SKILL_NAME: + return _execute_ad_hoc_relevance(content) + if skill_name == SUMMARIZATION_SKILL_NAME: + return _execute_ad_hoc_summarization(content) + if skill_name == RELATED_CONTENT_SKILL_NAME: + return _execute_ad_hoc_related_content(content) + raise ValueError(f"Unsupported skill name: {skill_name}") + + +def _execute_ad_hoc_classification(content: Content) -> SkillResult: + try: + classification = _execute_with_retries(CLASSIFICATION_SKILL_NAME, lambda: run_content_classification(content)) + content.content_type = classification["content_type"] + content.save(update_fields=["content_type"]) + if classification["confidence"] < settings.AI_CLASSIFICATION_REVIEW_THRESHOLD: + _upsert_review_queue_item( + content, + reason=ReviewReason.LOW_CONFIDENCE_CLASSIFICATION, + confidence=float(classification["confidence"]), + ) + return _create_skill_result( + content, + skill_name=CLASSIFICATION_SKILL_NAME, + status=SkillStatus.COMPLETED, + result_data=classification, + model_used=classification["model_used"], + latency_ms=classification["latency_ms"], + confidence=classification["confidence"], + ) + except Exception as exc: + return _create_failed_skill_result(content, skill_name=CLASSIFICATION_SKILL_NAME, error_message=str(exc)) + + +def _execute_ad_hoc_relevance(content: Content) -> SkillResult: + try: + relevance = _execute_with_retries(RELEVANCE_SKILL_NAME, lambda: run_relevance_scoring(content)) + relevance_score = float(relevance["relevance_score"]) + content.relevance_score = relevance_score + content.is_active = relevance_score >= settings.AI_RELEVANCE_REVIEW_THRESHOLD + content.save(update_fields=["relevance_score", "is_active"]) + if settings.AI_RELEVANCE_REVIEW_THRESHOLD <= relevance_score < settings.AI_RELEVANCE_SUMMARIZE_THRESHOLD: + _upsert_review_queue_item( + content, + reason=ReviewReason.BORDERLINE_RELEVANCE, + confidence=relevance_score, + ) + return _create_skill_result( + content, + skill_name=RELEVANCE_SKILL_NAME, + status=SkillStatus.COMPLETED, + result_data=relevance, + model_used=relevance["model_used"], + latency_ms=relevance["latency_ms"], + confidence=relevance_score, + ) + except Exception as exc: + return _create_failed_skill_result(content, skill_name=RELEVANCE_SKILL_NAME, error_message=str(exc)) + + +def _execute_ad_hoc_summarization(content: Content) -> SkillResult: + try: + if (content.relevance_score or 0.0) < settings.AI_RELEVANCE_SUMMARIZE_THRESHOLD: + raise ValueError( + "Summarization requires relevance_score >= " + f"{settings.AI_RELEVANCE_SUMMARIZE_THRESHOLD:.2f}. Run relevance scoring first or review the content." + ) + summary = _execute_with_retries(SUMMARIZATION_SKILL_NAME, lambda: run_summarization(content)) + return _create_skill_result( + content, + skill_name=SUMMARIZATION_SKILL_NAME, + status=SkillStatus.COMPLETED, + result_data=summary, + model_used=summary["model_used"], + latency_ms=summary["latency_ms"], + ) + except Exception as exc: + return _create_failed_skill_result(content, skill_name=SUMMARIZATION_SKILL_NAME, error_message=str(exc)) + + +def _execute_ad_hoc_related_content(content: Content) -> SkillResult: + try: + matches = search_similar_content(content, limit=5, is_reference=False) + related_items = [_serialize_related_match(match) for match in matches] + top_score = max((item["score"] for item in related_items), default=None) + return _create_skill_result( + content, + skill_name=RELATED_CONTENT_SKILL_NAME, + status=SkillStatus.COMPLETED, + result_data={ + "related_items": related_items, + "limit": 5, + }, + model_used=f"embedding:{settings.EMBEDDING_MODEL}", + latency_ms=0, + confidence=top_score, + ) + except Exception as exc: + return _create_failed_skill_result(content, skill_name=RELATED_CONTENT_SKILL_NAME, error_message=str(exc)) + + def _execute_with_retries(skill_name: str, fn): last_exc: Exception | None = None for attempt in range(settings.AI_MAX_NODE_RETRIES + 1): @@ -289,6 +393,18 @@ def _execute_with_retries(skill_name: str, fn): raise last_exc +def _serialize_related_match(match: Any) -> dict[str, Any]: + payload = dict(getattr(match, "payload", {}) or {}) + return { + "content_id": payload.get("content_id"), + "title": payload.get("title"), + "url": payload.get("url"), + "published_date": payload.get("published_date"), + "source_plugin": payload.get("source_plugin"), + "score": float(getattr(match, "score", 0.0)), + } + + def _heuristic_classification(content: Content) -> dict[str, Any]: text = f"{content.title}\n{content.content_text}".lower() keyword_sets = { @@ -386,3 +502,13 @@ def _create_skill_result( previous.superseded_by = skill_result previous.save(update_fields=["superseded_by"]) return skill_result + + +def _create_failed_skill_result(content: Content, *, skill_name: str, error_message: str) -> SkillResult: + return _create_skill_result( + content, + skill_name=skill_name, + status=SkillStatus.FAILED, + result_data=None, + error_message=error_message, + ) diff --git a/core/tests/test_api.py b/core/tests/test_api.py index 33ed6e20..3e3134c3 100644 --- a/core/tests/test_api.py +++ b/core/tests/test_api.py @@ -1,3 +1,6 @@ +from types import SimpleNamespace +from unittest.mock import patch + from django.contrib.auth import get_user_model from django.urls import reverse from rest_framework import status @@ -185,6 +188,67 @@ def test_content_create_uses_tenant_from_url(self): created_content = Content.objects.get(title="New Content") self.assertEqual(created_content.tenant, self.owner_tenant) + @patch("core.pipeline.run_relevance_scoring") + def test_content_skill_action_runs_relevance_scoring(self, run_relevance_scoring_mock): + run_relevance_scoring_mock.return_value = { + "relevance_score": 0.82, + "explanation": "Strong match for the tenant topic.", + "used_llm": False, + "model_used": "embedding:test", + "latency_ms": 0, + } + + response = self.client.post( + f"/api/v1/tenants/{self.owner_tenant.id}/contents/{self.owner_content.id}/skills/relevance_scoring/", + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.owner_content.refresh_from_db() + self.assertEqual(self.owner_content.relevance_score, 0.82) + self.assertTrue(self.owner_content.is_active) + self.assertEqual(response.json()["skill_name"], "relevance_scoring") + self.assertEqual(response.json()["status"], SkillStatus.COMPLETED) + + def test_content_skill_action_records_failed_summary_when_relevance_is_too_low(self): + self.owner_content.relevance_score = 0.25 + self.owner_content.save(update_fields=["relevance_score"]) + + response = self.client.post( + f"/api/v1/tenants/{self.owner_tenant.id}/contents/{self.owner_content.id}/skills/summarization/", + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(response.json()["skill_name"], "summarization") + self.assertEqual(response.json()["status"], SkillStatus.FAILED) + self.assertIn("Summarization requires relevance_score", response.json()["error_message"]) + + @patch("core.pipeline.search_similar_content") + def test_content_skill_action_runs_find_related(self, search_similar_content_mock): + search_similar_content_mock.return_value = [ + SimpleNamespace( + score=0.91, + payload={ + "content_id": self.other_content.id, + "title": self.other_content.title, + "url": self.other_content.url, + "published_date": self.other_content.published_date, + "source_plugin": self.other_content.source_plugin, + }, + ) + ] + + response = self.client.post( + f"/api/v1/tenants/{self.owner_tenant.id}/contents/{self.owner_content.id}/skills/find_related/", + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(response.json()["skill_name"], "find_related") + self.assertEqual(response.json()["status"], SkillStatus.COMPLETED) + self.assertEqual(response.json()["result_data"]["related_items"][0]["content_id"], self.other_content.id) + def test_authenticated_nested_list_endpoints_smoke(self): list_endpoints = [ reverse("v1:tenant-config-list", kwargs={"tenant_id": self.owner_tenant.id}), diff --git a/docker-compose.yml b/docker-compose.yml index 464a834c..60f3056d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -98,7 +98,26 @@ services: volumes: - ./docker/nginx/default.conf:/etc/nginx/conf.d/default.conf:ro + frontend: + image: node:22-alpine + working_dir: /app/frontend + command: ["sh", "-lc", "npm install && npm run dev -- --hostname 0.0.0.0 --port 3000"] + env_file: + - .env + environment: + NEWSLETTER_API_BASE_URL: http://nginx + NEXT_TELEMETRY_DISABLED: "1" + depends_on: + nginx: + condition: service_started + ports: + - "3000:3000" + volumes: + - ./frontend:/app/frontend + - frontend_node_modules:/app/frontend/node_modules + volumes: postgres_data: redis_data: qdrant_data: + frontend_node_modules: diff --git a/frontend/.env.example b/frontend/.env.example new file mode 100644 index 00000000..fc320852 --- /dev/null +++ b/frontend/.env.example @@ -0,0 +1,5 @@ +# Copy this file to .env.local when running the Next.js app outside Docker. +NEWSLETTER_API_BASE_URL=http://127.0.0.1:8080 +NEWSLETTER_API_USERNAME=admin +NEWSLETTER_API_PASSWORD=adminpass +NEXT_TELEMETRY_DISABLED=1 \ No newline at end of file diff --git a/frontend/.eslintrc b/frontend/.eslintrc new file mode 100644 index 00000000..e69de29b diff --git a/frontend/.prettierrc b/frontend/.prettierrc new file mode 100644 index 00000000..e69de29b diff --git a/frontend/app/admin/health/page.tsx b/frontend/app/admin/health/page.tsx new file mode 100644 index 00000000..a0b6a92d --- /dev/null +++ b/frontend/app/admin/health/page.tsx @@ -0,0 +1,108 @@ +import { AppShell } from "@/components/app-shell"; +import { StatusBadge } from "@/components/status-badge"; +import { getTenantIngestionRuns, getTenantSourceConfigs, getTenants } from "@/lib/api"; +import type { HealthStatus } from "@/lib/types"; +import { formatDate, healthTone, selectTenant } from "@/lib/view-helpers"; + +type HealthPageProps = { + searchParams: Promise>; +}; + +function deriveSourceStatus(isActive: boolean, latestRunStatus: string | null, lastFetchedAt: string | null): HealthStatus { + if (!isActive) { + return "idle"; + } + if (latestRunStatus === "failed") { + return "failing"; + } + if (latestRunStatus === "running") { + return "degraded"; + } + if (!lastFetchedAt) { + return "degraded"; + } + return "healthy"; +} + +export default async function HealthPage({ searchParams }: HealthPageProps) { + const resolvedSearchParams = await searchParams; + const tenants = await getTenants(); + const selectedTenant = selectTenant(tenants, resolvedSearchParams); + + if (!selectedTenant) { + return ( + +
Create a tenant first in Django admin.
+ + ); + } + + const [sourceConfigs, ingestionRuns] = await Promise.all([ + getTenantSourceConfigs(selectedTenant.id), + getTenantIngestionRuns(selectedTenant.id), + ]); + + const latestRunByPlugin = new Map(); + for (const ingestionRun of ingestionRuns) { + if (!latestRunByPlugin.has(ingestionRun.plugin_name)) { + latestRunByPlugin.set(ingestionRun.plugin_name, ingestionRun); + } + } + + return ( + +
+ + + + + + + + + + + + + {sourceConfigs.length === 0 ? ( + + + + ) : null} + {sourceConfigs.map((sourceConfig) => { + const latestRun = latestRunByPlugin.get(sourceConfig.plugin_name) ?? null; + const status = deriveSourceStatus(sourceConfig.is_active, latestRun?.status ?? null, sourceConfig.last_fetched_at); + return ( + + + + + + + + + ); + })} + +
SourceStatusLast fetchLatest runItemsErrors
+
No source configurations exist for this tenant yet.
+
+ {sourceConfig.plugin_name} +
+ Config #{sourceConfig.id} + {sourceConfig.is_active ? "active" : "disabled"} +
+ 		+ {status} + {formatDate(sourceConfig.last_fetched_at)}{latestRun ? `${latestRun.status} at ${formatDate(latestRun.started_at)}` : "No runs yet"} + {latestRun ? `${latestRun.items_ingested}/${latestRun.items_fetched}` : "0/0"} + {latestRun?.error_message || "-"}
+
+ + ); +} \ No newline at end of file diff --git a/frontend/app/admin/sources/page.tsx b/frontend/app/admin/sources/page.tsx new file mode 100644 index 00000000..2eff464b --- /dev/null +++ b/frontend/app/admin/sources/page.tsx @@ -0,0 +1,132 @@ +import { AppShell } from "@/components/app-shell"; +import { StatusBadge } from "@/components/status-badge"; +import { getTenantIngestionRuns, getTenantSourceConfigs, getTenants } from "@/lib/api"; +import { formatDate, getErrorMessage, getSuccessMessage, selectTenant } from "@/lib/view-helpers"; + +type SourcesPageProps = { + searchParams: Promise>; +}; + +export default async function SourcesPage({ searchParams }: SourcesPageProps) { + const resolvedSearchParams = await searchParams; + const tenants = await getTenants(); + const selectedTenant = selectTenant(tenants, resolvedSearchParams); + + if (!selectedTenant) { + return ( + +
Create a tenant first in Django admin.
+ + ); + } + + const [sourceConfigs, ingestionRuns] = await Promise.all([ + getTenantSourceConfigs(selectedTenant.id), + getTenantIngestionRuns(selectedTenant.id), + ]); + const latestRunByPlugin = new Map(); + for (const ingestionRun of ingestionRuns) { + if (!latestRunByPlugin.has(ingestionRun.plugin_name)) { + latestRunByPlugin.set(ingestionRun.plugin_name, ingestionRun); + } + } + + const errorMessage = getErrorMessage(resolvedSearchParams); + const successMessage = getSuccessMessage(resolvedSearchParams); + + return ( + + {errorMessage ?
{errorMessage}
: null} + {successMessage ?
{successMessage}
: null} + +
+
+

Add source

+
+ + + +