diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index cb48e154..8cbb3439 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -40,4 +40,4 @@ jobs: run: pre-commit install --install-hooks - name: Run lint - run: just lint \ No newline at end of file + run: just lint diff --git a/.gitignore b/.gitignore index d7eafb7c..639cdcc9 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,4 @@ celerybeat-schedule* db.sqlite3 staticfiles/ -docs/ \ No newline at end of file +docs/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9a6c0a53..a2512cf8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,4 +12,4 @@ repos: name: just lint entry: just lint language: system - pass_filenames: false \ No newline at end of file + pass_filenames: false diff --git a/core/api.py b/core/api.py index c433fdc3..46782db9 100644 --- a/core/api.py +++ b/core/api.py @@ -1,6 +1,14 @@ from typing import Any -from rest_framework import viewsets +from drf_spectacular.utils import ( + OpenApiExample, + OpenApiParameter, + OpenApiResponse, + extend_schema, + extend_schema_view, + inline_serializer, +) +from rest_framework import serializers, viewsets from rest_framework.exceptions import NotFound from core.models import ( @@ -26,6 +34,365 @@ UserFeedbackSerializer, ) +TENANT_ID_PARAMETER = OpenApiParameter( + name="tenant_id", + type=int, + location=OpenApiParameter.PATH, + description="The unique ID of the tenant that owns this nested resource.", +) + +TENANT_CREATE_REQUEST_EXAMPLE = OpenApiExample( + "Create Tenant Request", + value={ + "name": "AI Weekly", + "topic_description": "Coverage of developer tools, model releases, and applied AI workflows.", + "content_retention_days": 180, + }, + request_only=True, +) + +TENANT_RESPONSE_EXAMPLE = OpenApiExample( + "Tenant Response", + value={ + "id": 1, + "name": "AI Weekly", + "user": 7, + "topic_description": "Coverage of developer tools, model releases, and applied AI workflows.", + "content_retention_days": 180, + "created_at": "2026-04-26T12:00:00Z", + }, + response_only=True, +) + +SOURCE_CONFIG_CREATE_REQUEST_EXAMPLE = OpenApiExample( + "Create RSS Source Request", + value={ + "plugin_name": "rss", + "config": { + "feed_url": "https://example.com/feed.xml", + }, + "is_active": True, + }, + request_only=True, +) + +SOURCE_CONFIG_REDDIT_REQUEST_EXAMPLE = OpenApiExample( + "Create Reddit Source Request", + value={ + "plugin_name": "reddit", + "config": { + "subreddit": "MachineLearning", + "listing": "both", + "limit": 25, + }, + "is_active": True, + }, + request_only=True, +) + +SOURCE_CONFIG_RESPONSE_EXAMPLE = OpenApiExample( + "Source Configuration Response", + value={ + "id": 12, + "tenant": 1, + "plugin_name": "rss", + "config": { + "feed_url": "https://example.com/feed.xml", + }, + "is_active": True, + "last_fetched_at": "2026-04-26T12:30:00Z", + }, + response_only=True, +) + +CONTENT_CREATE_REQUEST_EXAMPLE = OpenApiExample( + "Create Content Request", + value={ + "url": "https://example.com/posts/agent-memory-patterns", + "title": "Practical Agent Memory Patterns", + "author": "Jane Doe", + "entity": 4, + "source_plugin": "rss", + "content_type": "article", + "published_date": "2026-04-25T14:00:00Z", + "content_text": "A walkthrough of short-term and long-term memory patterns for production agents.", + "relevance_score": 0.92, + "is_reference": False, + "is_active": True, + }, + request_only=True, +) + +CONTENT_RESPONSE_EXAMPLE = OpenApiExample( + "Content Response", + value={ + "id": 44, + "tenant": 1, + "url": "https://example.com/posts/agent-memory-patterns", + "title": "Practical Agent Memory Patterns", + "author": "Jane Doe", + "entity": 4, + "source_plugin": "rss", + "content_type": "article", + "published_date": "2026-04-25T14:00:00Z", + "ingested_at": "2026-04-26T12:05:00Z", + "content_text": "A walkthrough of short-term and long-term memory patterns for production agents.", + "relevance_score": 0.92, + "embedding_id": "emb_01jabcxyz", + "is_reference": False, + "is_active": True, + }, + response_only=True, +) + +SKILL_RESULT_RESPONSE_EXAMPLE = OpenApiExample( + "Skill Result Response", + value={ + "id": 91, + "content": 44, + "tenant": 1, + "skill_name": "relevance_classifier", + "status": "completed", + "result_data": { + "label": "high_relevance", + "reasoning": "The article directly covers agent memory design patterns.", + }, + "error_message": "", + "model_used": "gpt-4.1-mini", + "latency_ms": 842, + "confidence": 0.97, + "created_at": "2026-04-26T12:06:00Z", + "superseded_by": None, + }, + response_only=True, +) + +AUTHENTICATION_REQUIRED_EXAMPLE = OpenApiExample( + "Authentication Required", + value={ + "type": "client_error", + "errors": [ + { + "code": "not_authenticated", + "detail": "Authentication credentials were not provided.", + "attr": None, + } + ], + }, + response_only=True, + status_codes=["403"], +) + +AUTHENTICATION_REQUIRED_RESPONSE = OpenApiResponse( + response=inline_serializer( + name="AuthenticationRequiredResponse", + fields={ + "type": serializers.CharField(), + "errors": inline_serializer( + name="AuthenticationRequiredError", + fields={ + "code": serializers.CharField(), + "detail": serializers.CharField(), + "attr": serializers.CharField(allow_null=True), + }, + many=True, + ), + }, + ), + description="Authentication credentials are required to access this endpoint.", + examples=[AUTHENTICATION_REQUIRED_EXAMPLE], +) + + +def build_success_response(response, description: str, examples: list[OpenApiExample] | None = None): + response_kwargs = { + "response": response, + "description": description, + } + if examples is not None: + response_kwargs["examples"] = examples + return OpenApiResponse(**response_kwargs) + + +def build_crud_action_overrides( + serializer_class, + resource_plural: str, + resource_singular: str, + *, + list_examples: list[OpenApiExample] | None = None, + retrieve_examples: list[OpenApiExample] | None = None, + create_examples: list[OpenApiExample] | None = None, + create_response_examples: list[OpenApiExample] | None = None, +): + overrides: dict[str, dict[str, Any]] = { + "list": { + "responses": { + 200: build_success_response( + serializer_class(many=True), + f"A list of {resource_plural}.", + examples=list_examples if list_examples is not None else [], + ), + 403: AUTHENTICATION_REQUIRED_RESPONSE, + } + }, + "retrieve": { + "responses": { + 200: build_success_response( + serializer_class, + f"The requested {resource_singular}.", + examples=retrieve_examples, + ), + 403: AUTHENTICATION_REQUIRED_RESPONSE, + } + }, + "create": { + "responses": { + 201: build_success_response( + serializer_class, + f"The newly created {resource_singular}.", + examples=create_response_examples, + ), + 403: AUTHENTICATION_REQUIRED_RESPONSE, + } + }, + "update": { + "responses": { + 200: build_success_response(serializer_class, f"The updated {resource_singular}."), + 403: AUTHENTICATION_REQUIRED_RESPONSE, + } + }, + "partial_update": { + "responses": { + 200: build_success_response(serializer_class, f"The updated {resource_singular}."), + 403: AUTHENTICATION_REQUIRED_RESPONSE, + } + }, + "destroy": { + "responses": { + 204: OpenApiResponse(description=f"The {resource_singular} was deleted."), + 403: AUTHENTICATION_REQUIRED_RESPONSE, + } + }, + } + if create_examples: + overrides["create"]["examples"] = create_examples + return overrides + + +def document_user_owned_viewset( + resource_plural: str, + resource_singular: str, + create_description: str, + tag: str, + action_overrides: dict[str, dict] | None = None, +): + action_overrides = action_overrides or {} + + def schema(action: str, **kwargs): + schema_kwargs = {"tags": [tag], **kwargs} + action_override = action_overrides.get(action, {}) + override_responses = action_override.get("responses", {}) + if override_responses: + responses = dict(schema_kwargs.get("responses", {})) + responses.update(override_responses) + schema_kwargs["responses"] = responses + schema_kwargs.update({key: value for key, value in action_override.items() if key != "responses"}) + return extend_schema(**schema_kwargs) + + return extend_schema_view( + list=schema( + "list", + summary=f"List {resource_plural}", + description=f"Return all {resource_plural} owned by the authenticated user.", + ), + retrieve=schema( + "retrieve", + summary=f"Get {resource_singular}", + description=f"Return a single {resource_singular} owned by the authenticated user.", + ), + create=schema( + "create", + summary=f"Create {resource_singular}", + description=create_description, + ), + update=schema( + "update", + summary=f"Replace {resource_singular}", + description=f"Replace an existing {resource_singular} owned by the authenticated user.", + ), + partial_update=schema( + "partial_update", + summary=f"Update {resource_singular}", + description=f"Update one or more fields on an existing {resource_singular} owned by the authenticated user.", + ), + destroy=schema( + "destroy", + summary=f"Delete {resource_singular}", + description=f"Delete an existing {resource_singular} owned by the authenticated user.", + ), + ) + + +def document_tenant_owned_viewset( + resource_plural: str, + resource_singular: str, + create_description: str, + tag: str, + action_overrides: dict[str, dict] | None = None, +): + parameters = [TENANT_ID_PARAMETER] + action_overrides = action_overrides or {} + + def schema(action: str, **kwargs): + schema_kwargs = {"tags": [tag], **kwargs} + action_override = action_overrides.get(action, {}) + override_responses = action_override.get("responses", {}) + if override_responses: + responses = dict(schema_kwargs.get("responses", {})) + responses.update(override_responses) + schema_kwargs["responses"] = responses + schema_kwargs.update({key: value for key, value in action_override.items() if key != "responses"}) + return extend_schema(**schema_kwargs) + + return extend_schema_view( + list=schema( + "list", + summary=f"List {resource_plural}", + description=f"Return all {resource_plural} for the selected tenant.", + parameters=parameters, + ), + retrieve=schema( + "retrieve", + summary=f"Get {resource_singular}", + description=f"Return a single {resource_singular} for the selected tenant.", + parameters=parameters, + ), + create=schema( + "create", + summary=f"Create {resource_singular}", + description=create_description, + parameters=parameters, + ), + update=schema( + "update", + summary=f"Replace {resource_singular}", + description=f"Replace an existing {resource_singular} for the selected tenant.", + parameters=parameters, + ), + partial_update=schema( + "partial_update", + summary=f"Update {resource_singular}", + description=f"Update one or more fields on an existing {resource_singular} for the selected tenant.", + parameters=parameters, + ), + destroy=schema( + "destroy", + summary=f"Delete {resource_singular}", + description=f"Delete an existing {resource_singular} for the selected tenant.", + parameters=parameters, + ), + ) + class TenantOwnedQuerysetMixin: queryset: Any = None @@ -54,9 +421,25 @@ def perform_create(self, serializer): serializer.save(tenant=self.get_tenant()) + +@document_user_owned_viewset( + resource_plural="tenants", + resource_singular="tenant", + create_description="Create a new tenant for the authenticated user. The requesting user is attached automatically.", + tag="Tenant Management", + action_overrides=build_crud_action_overrides( + TenantSerializer, + resource_plural="tenants owned by the authenticated user", + resource_singular="tenant", + create_examples=[TENANT_CREATE_REQUEST_EXAMPLE, TENANT_RESPONSE_EXAMPLE], + create_response_examples=[TENANT_RESPONSE_EXAMPLE], + retrieve_examples=[TENANT_RESPONSE_EXAMPLE], + ), +) class TenantViewSet(viewsets.ModelViewSet): serializer_class = TenantSerializer queryset = Tenant.objects.select_related("user") + lookup_url_kwarg = "id" def get_queryset(self): return self.queryset.filter(user=self.request.user) @@ -65,26 +448,85 @@ def perform_create(self, serializer): serializer.save(user=self.request.user) +@document_tenant_owned_viewset( + resource_plural="tenant configurations", + resource_singular="tenant configuration", + create_description="Create a new tenant configuration record for the selected tenant, including authority weighting and decay settings.", + tag="Tenant Management", + action_overrides=build_crud_action_overrides( + TenantConfigSerializer, + resource_plural="tenant configurations for the selected tenant", + resource_singular="tenant configuration", + ), +) class TenantConfigViewSet(TenantOwnedQuerysetMixin, viewsets.ModelViewSet): serializer_class = TenantConfigSerializer queryset = TenantConfig.objects.select_related("tenant") +@document_tenant_owned_viewset( + resource_plural="entities", + resource_singular="entity", + create_description="Create a new tracked entity for the selected tenant, such as a company, person, or project.", + tag="Entity Catalog", + action_overrides=build_crud_action_overrides( + EntitySerializer, + resource_plural="entities for the selected tenant", + resource_singular="entity", + ), +) class EntityViewSet(TenantOwnedQuerysetMixin, viewsets.ModelViewSet): serializer_class = EntitySerializer queryset = Entity.objects.select_related("tenant") +@document_tenant_owned_viewset( + resource_plural="content items", + resource_singular="content item", + create_description="Create a new content item for the selected tenant. Any related entity must belong to the same tenant.", + tag="Content Library", + action_overrides=build_crud_action_overrides( + ContentSerializer, + resource_plural="content items for the selected tenant", + resource_singular="content item", + create_examples=[CONTENT_CREATE_REQUEST_EXAMPLE, CONTENT_RESPONSE_EXAMPLE], + create_response_examples=[CONTENT_RESPONSE_EXAMPLE], + retrieve_examples=[CONTENT_RESPONSE_EXAMPLE], + ), +) class ContentViewSet(TenantOwnedQuerysetMixin, viewsets.ModelViewSet): serializer_class = ContentSerializer queryset = Content.objects.select_related("tenant", "entity") +@document_tenant_owned_viewset( + resource_plural="skill results", + resource_singular="skill result", + create_description="Create a new skill result for tenant content. The referenced content must belong to the selected tenant.", + tag="AI Processing", + action_overrides=build_crud_action_overrides( + SkillResultSerializer, + resource_plural="skill results for the selected tenant", + resource_singular="skill result", + retrieve_examples=[SKILL_RESULT_RESPONSE_EXAMPLE], + ), +) class SkillResultViewSet(TenantOwnedQuerysetMixin, viewsets.ModelViewSet): serializer_class = SkillResultSerializer queryset = SkillResult.objects.select_related("content", "tenant", "superseded_by") +@document_tenant_owned_viewset( + resource_plural="user feedback entries", + resource_singular="user feedback entry", + create_description="Create a new feedback entry for content in the selected tenant. The authenticated user is recorded automatically.", + tag="Feedback", + action_overrides=build_crud_action_overrides( + UserFeedbackSerializer, + resource_plural="user feedback entries for the selected tenant", + resource_singular="user feedback entry", + ), +) class UserFeedbackViewSet(TenantOwnedQuerysetMixin, viewsets.ModelViewSet): serializer_class = UserFeedbackSerializer queryset = UserFeedback.objects.select_related("content", "tenant", "user") @@ -93,16 +535,56 @@ def perform_create(self, serializer): serializer.save(tenant=self.get_tenant(), user=self.request.user) +@document_tenant_owned_viewset( + resource_plural="ingestion runs", + resource_singular="ingestion run", + create_description="Create a new ingestion run record for the selected tenant to track a content ingestion attempt and its status.", + tag="Ingestion", + action_overrides=build_crud_action_overrides( + IngestionRunSerializer, + resource_plural="ingestion runs for the selected tenant", + resource_singular="ingestion run", + ), +) class IngestionRunViewSet(TenantOwnedQuerysetMixin, viewsets.ModelViewSet): serializer_class = IngestionRunSerializer queryset = IngestionRun.objects.select_related("tenant") +@document_tenant_owned_viewset( + resource_plural="source configurations", + resource_singular="source configuration", + create_description="Create a new source configuration for the selected tenant. Plugin-specific configuration is validated before the record is saved.", + tag="Ingestion", + action_overrides=build_crud_action_overrides( + SourceConfigSerializer, + resource_plural="source configurations for the selected tenant", + resource_singular="source configuration", + create_examples=[ + SOURCE_CONFIG_CREATE_REQUEST_EXAMPLE, + SOURCE_CONFIG_REDDIT_REQUEST_EXAMPLE, + SOURCE_CONFIG_RESPONSE_EXAMPLE, + ], + create_response_examples=[SOURCE_CONFIG_RESPONSE_EXAMPLE], + retrieve_examples=[SOURCE_CONFIG_RESPONSE_EXAMPLE], + ), +) class SourceConfigViewSet(TenantOwnedQuerysetMixin, viewsets.ModelViewSet): serializer_class = SourceConfigSerializer queryset = SourceConfig.objects.select_related("tenant") +@document_tenant_owned_viewset( + resource_plural="review queue entries", + resource_singular="review queue entry", + create_description="Create a new review queue entry for the selected tenant. The referenced content must belong to the same tenant.", + tag="Review Queue", + action_overrides=build_crud_action_overrides( + ReviewQueueSerializer, + resource_plural="review queue entries for the selected tenant", + resource_singular="review queue entry", + ), +) class ReviewQueueViewSet(TenantOwnedQuerysetMixin, viewsets.ModelViewSet): serializer_class = ReviewQueueSerializer queryset = ReviewQueue.objects.select_related("content", "tenant") diff --git a/core/api_urls.py b/core/api_urls.py index 496a4cca..df7532c1 100644 --- a/core/api_urls.py +++ b/core/api_urls.py @@ -1,5 +1,5 @@ -from django.urls import path from rest_framework.routers import DefaultRouter +from rest_framework_nested.routers import NestedSimpleRouter from core.api import ( ContentViewSet, @@ -18,74 +18,17 @@ router = DefaultRouter() router.register("tenants", TenantViewSet, basename="tenant") -tenant_config_list = TenantConfigViewSet.as_view({"get": "list", "post": "create"}) -tenant_config_detail = TenantConfigViewSet.as_view( - {"get": "retrieve", "put": "update", "patch": "partial_update", "delete": "destroy"} -) - -entity_list = EntityViewSet.as_view({"get": "list", "post": "create"}) -entity_detail = EntityViewSet.as_view( - {"get": "retrieve", "put": "update", "patch": "partial_update", "delete": "destroy"} -) - -content_list = ContentViewSet.as_view({"get": "list", "post": "create"}) -content_detail = ContentViewSet.as_view( - {"get": "retrieve", "put": "update", "patch": "partial_update", "delete": "destroy"} -) - -skill_result_list = SkillResultViewSet.as_view({"get": "list", "post": "create"}) -skill_result_detail = SkillResultViewSet.as_view( - {"get": "retrieve", "put": "update", "patch": "partial_update", "delete": "destroy"} -) - -feedback_list = UserFeedbackViewSet.as_view({"get": "list", "post": "create"}) -feedback_detail = UserFeedbackViewSet.as_view( - {"get": "retrieve", "put": "update", "patch": "partial_update", "delete": "destroy"} -) - -ingestion_run_list = IngestionRunViewSet.as_view({"get": "list", "post": "create"}) -ingestion_run_detail = IngestionRunViewSet.as_view( - {"get": "retrieve", "put": "update", "patch": "partial_update", "delete": "destroy"} -) - -source_config_list = SourceConfigViewSet.as_view({"get": "list", "post": "create"}) -source_config_detail = SourceConfigViewSet.as_view( - {"get": "retrieve", "put": "update", "patch": "partial_update", "delete": "destroy"} -) - -review_queue_list = ReviewQueueViewSet.as_view({"get": "list", "post": "create"}) -review_queue_detail = ReviewQueueViewSet.as_view( - {"get": "retrieve", "put": "update", "patch": "partial_update", "delete": "destroy"} -) +tenant_router = NestedSimpleRouter(router, r"tenants", lookup="tenant") +tenant_router.register(r"tenant-configs", TenantConfigViewSet, basename="tenant-config") +tenant_router.register(r"entities", EntityViewSet, basename="tenant-entity") +tenant_router.register(r"contents", ContentViewSet, basename="tenant-content") +tenant_router.register(r"skill-results", SkillResultViewSet, basename="tenant-skill-result") +tenant_router.register(r"feedback", UserFeedbackViewSet, basename="tenant-feedback") +tenant_router.register(r"ingestion-runs", IngestionRunViewSet, basename="tenant-ingestion-run") +tenant_router.register(r"source-configs", SourceConfigViewSet, basename="tenant-source-config") +tenant_router.register(r"review-queue", ReviewQueueViewSet, basename="tenant-review-queue") urlpatterns = [ *router.urls, - path("tenants//tenant-configs/", tenant_config_list, name="tenant-config-list"), - path("tenants//tenant-configs//", tenant_config_detail, name="tenant-config-detail"), - path("tenants//entities/", entity_list, name="tenant-entity-list"), - path("tenants//entities//", entity_detail, name="tenant-entity-detail"), - path("tenants//contents/", content_list, name="tenant-content-list"), - path("tenants//contents//", content_detail, name="tenant-content-detail"), - path("tenants//skill-results/", skill_result_list, name="tenant-skill-result-list"), - path("tenants//skill-results//", skill_result_detail, name="tenant-skill-result-detail"), - path("tenants//feedback/", feedback_list, name="tenant-feedback-list"), - path("tenants//feedback//", feedback_detail, name="tenant-feedback-detail"), - path("tenants//ingestion-runs/", ingestion_run_list, name="tenant-ingestion-run-list"), - path( - "tenants//ingestion-runs//", - ingestion_run_detail, - name="tenant-ingestion-run-detail", - ), - path("tenants//source-configs/", source_config_list, name="tenant-source-config-list"), - path( - "tenants//source-configs//", - source_config_detail, - name="tenant-source-config-detail", - ), - path("tenants//review-queue/", review_queue_list, name="tenant-review-queue-list"), - path( - "tenants//review-queue//", - review_queue_detail, - name="tenant-review-queue-detail", - ), + *tenant_router.urls, ] diff --git a/core/favicon.ico b/core/favicon.ico deleted file mode 100644 index 02d0115b..00000000 Binary files a/core/favicon.ico and /dev/null differ diff --git a/core/static/core/favicon.ico b/core/static/core/favicon.ico index b67ed8ce..02d0115b 100644 Binary files a/core/static/core/favicon.ico and b/core/static/core/favicon.ico differ diff --git a/core/logo.svg b/core/static/core/logo.svg similarity index 97% rename from core/logo.svg rename to core/static/core/logo.svg index 887a89ba..0c20a100 100644 --- a/core/logo.svg +++ b/core/static/core/logo.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/core/templates/admin/index.html b/core/templates/admin/index.html index 3b785045..04f14d9b 100644 --- a/core/templates/admin/index.html +++ b/core/templates/admin/index.html @@ -1,16 +1,10 @@ {% extends "unfold/layouts/base.html" %} - {% block content %}
-

- Average Authority Weight -

-

- {{ avg_authority_weight }} -

+

Average Authority Weight

+

{{ avg_authority_weight }}

- {{ block.super }} {% endblock %} diff --git a/core/tests/test_api.py b/core/tests/test_api.py index 578b1ac7..33ed6e20 100644 --- a/core/tests/test_api.py +++ b/core/tests/test_api.py @@ -94,6 +94,30 @@ def setUp(self): ) self.client.force_authenticate(self.owner) + def assert_standardized_validation_error(self, payload, attr): + self.assertEqual(payload["type"], "validation_error") + self.assertTrue(any(error["attr"] == attr for error in payload["errors"])) + + def test_tenant_list_requires_authentication(self): + self.client.force_authenticate(user=None) + + response = self.client.get(reverse("v1:tenant-list"), HTTP_HOST="localhost") + + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + self.assertEqual( + response.json(), + { + "type": "client_error", + "errors": [ + { + "code": "not_authenticated", + "detail": "Authentication credentials were not provided.", + "attr": None, + } + ], + }, + ) + def test_tenant_list_is_scoped_to_request_user(self): response = self.client.get(reverse("v1:tenant-list")) @@ -139,7 +163,7 @@ def test_feedback_rejects_cross_tenant_content(self): ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertIn("content", response.json()) + self.assert_standardized_validation_error(response.json(), "content") def test_content_create_uses_tenant_from_url(self): response = self.client.post( @@ -236,4 +260,4 @@ def test_source_config_create_validates_plugin_config(self): ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertIn("config", response.json()) + self.assert_standardized_validation_error(response.json(), "config") diff --git a/core/utils.py b/core/utils.py index a9d57942..69800201 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1,6 +1,8 @@ from django.db.models import Avg + from .models import TenantConfig + def dashboard_callback(request, context): # Calculate the average authority weight across all tenants avg_weight = TenantConfig.objects.aggregate(Avg('upvote_authority_weight'))['upvote_authority_weight__avg'] @@ -9,4 +11,4 @@ def dashboard_callback(request, context): context.update({ "avg_authority_weight": round(avg_weight, 2) if avg_weight else 0, }) - return context \ No newline at end of file + return context diff --git a/docker-compose.yml b/docker-compose.yml index f4e536dd..464a834c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,6 @@ x-app-common: &app-common - .env volumes: - .:/app - - static_data:/app/staticfiles x-shared-env: &shared-env DATABASE_URL: postgresql://newsletter:newsletter@postgres:5432/newsletter_maker @@ -21,7 +20,7 @@ services: environment: BOOTSTRAP_APP: "true" <<: *shared-env - command: ["gunicorn", "newsletter_maker.wsgi:application", "--bind", "0.0.0.0:8000", "--workers", "2"] + command: ["gunicorn", "newsletter_maker.wsgi:application", "--bind", "0.0.0.0:8000", "--workers", "2", "--reload"] depends_on: postgres: condition: service_healthy @@ -40,7 +39,7 @@ services: environment: BOOTSTRAP_APP: "false" <<: *shared-env - command: ["celery", "-A", "newsletter_maker", "worker", "--loglevel=info"] + command: ["watchmedo", "auto-restart", "--directory=.", "--pattern=*.py", "--recursive", "--", "celery", "-A", "newsletter_maker", "worker", "--loglevel=info"] depends_on: postgres: condition: service_healthy @@ -52,7 +51,7 @@ services: environment: BOOTSTRAP_APP: "false" <<: *shared-env - command: ["celery", "-A", "newsletter_maker", "beat", "--loglevel=info"] + command: ["watchmedo", "auto-restart", "--directory=.", "--pattern=*.py", "--recursive", "--", "celery", "-A", "newsletter_maker", "beat", "--loglevel=info"] depends_on: postgres: condition: service_healthy @@ -98,10 +97,8 @@ services: - "8080:80" volumes: - ./docker/nginx/default.conf:/etc/nginx/conf.d/default.conf:ro - - static_data:/var/www/static:ro volumes: postgres_data: redis_data: qdrant_data: - static_data: diff --git a/docker/nginx/default.conf b/docker/nginx/default.conf index 706deb18..4f6e0a6f 100644 --- a/docker/nginx/default.conf +++ b/docker/nginx/default.conf @@ -4,10 +4,6 @@ server { client_max_body_size 20m; - location /static/ { - alias /var/www/static/; - } - location / { proxy_pass http://django:8000; proxy_set_header Host $host; diff --git a/docker/web/Dockerfile b/docker/web/Dockerfile index fb12f127..e4576725 100644 --- a/docker/web/Dockerfile +++ b/docker/web/Dockerfile @@ -10,8 +10,9 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* COPY requirements.txt ./ -RUN pip install --no-cache-dir --upgrade pip \ - && pip install --no-cache-dir -r requirements.txt +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --upgrade pip && \ + pip install -r requirements.txt COPY . . diff --git a/docker/web/entrypoint.sh b/docker/web/entrypoint.sh index a607bb85..967d5c9d 100644 --- a/docker/web/entrypoint.sh +++ b/docker/web/entrypoint.sh @@ -3,7 +3,6 @@ set -eu if [ "${BOOTSTRAP_APP:-false}" = "true" ]; then python manage.py migrate --noinput - python manage.py collectstatic --noinput if [ "${DJANGO_SUPERUSER_USERNAME:-}" ] && [ "${DJANGO_SUPERUSER_EMAIL:-}" ] && [ "${DJANGO_SUPERUSER_PASSWORD:-}" ]; then python manage.py shell <<'PY' diff --git a/docs/FILES.md b/docs/FILES.md deleted file mode 100644 index 8750e75a..00000000 --- a/docs/FILES.md +++ /dev/null @@ -1,368 +0,0 @@ -# File Organization - -In Django, you have one **project** (the container for your settings and main routing) and one or more **apps** (reusable modules that handle specific logic). - -## File Breakdown: `newsletter_maker/` (The Project) - -This is the "brain." It contains your global settings, main URL configuration, and WSGI/ASGI entry points for the server. - -- **`settings.py`**: All configuration (Database URLs, installed apps, Celery broker settings). -- **`urls.py`**: The "main" URL file that imports the `api_urls.py` and `core.urls`. -- **`wsgi.py` / `asgi.py`**: The interface Gunicorn uses to run your app. -- **`celery.py`**: Where Celery is initialized for the project. - -## File Breakdown: `core/` (The Application) - -This is where the actual "features" live. Django encourages putting logic into apps so you could, in theory, pluck this `core` folder out and drop it into a different project. The `core` name is a popular label for the app that houses "base" functionality—like custom user models, global tasks, or shared logic—that doesn't fit neatly into a more specific feature name. - -- **`models.py`**: The most important file. It defines your database schema using Python classes. -- **`serializers.py`**: Part of **DRF** (Django REST Framework). It converts your `models.py` data into JSON for the API. -- **`tasks.py`**: Contains your **Celery** background tasks (e.g., the actual code that sends the newsletter). -- **`api.py`**: This file DRF logic contains **ViewSets** or **Views**. It defines the behavior of the API—such as how it queries the database, applies permissions, and uses serializers to format data. -- **`api_urls.py`**: : This file contains the URL patterns specific to your API. It maps the incoming URL paths (like `/api/v1/newsletter/`) to the logic defined in `api.py`. -- **`admin.py`**: Configures how your models look in the built-in Django `/admin` interface. -- **`views.py` & `urls.py`**: Handle standard web requests and map them to templates. -- **`embeddings.py`**: Likely a custom file for your specific app (given the `qdrant` service you have), probably handling Vector Search or AI logic. -- **`migrations/`**: A history of your database changes. -- **`templates/` & `static/`**: Your HTML files and CSS/JS/images. -- **`management/`**: Contains custom terminal commands (e.g., `python manage.py my_custom_command`). -- **`tests.py`**: Where your automated tests live. - -## Simplifying `api.py` - -Your file is quite long because you are manually mapping 8 different ViewSets. If you want to keep the **Nested URL** structure (`/tenants/1/entities/`) but use a Router to save space, the most popular tool is a library called **`drf-nested-routers`**. This loses the benefit of seeing endpoints at a glance. - -```python -# With drf-nested-routers (Simplified concept) -tenant_router = NestedSimpleRouter(router, r'tenants', lookup='tenant') -tenant_router.register(r'entities', EntityViewSet, basename='tenant-entities') -``` - -### API Documentation (The Professional Way) - -What is the actual endpoint for `router.register("tenants", ...)`? - -The `DefaultRouter` generates two main URL patterns for every registered string: - -1. **The List View**: `/api/v1/tenants/` - - Maps to `GET` (list) and `POST` (create). -2. **The Detail View**: `/api/v1/tenants//` - - Maps to `GET` (retrieve), `PUT` (update), `PATCH`, and `DELETE`. - -Because you used `DefaultRouter`, it also adds a **Root View** at `/api/v1/` that acts as a directory for all your registered endpoints. - -For larger projects, developers use **drf-spectacular** to automatically generate a **Swagger** or **Redoc** page (usually at `/api/docs/`). - -This creates a beautiful, interactive documentation site that shows every endpoint, the required parameters (like `tenant_id`), and the expected JSON format. - -**Step 1: Install the Package** - -First, install the library and its optional dependencies for the Swagger UI. Since you are using Docker, you can run this command to install it temporarily or add it to your `requirements.txt`: - -```bash -docker exec -it newsletter-maker-django-1 pip install drf-spectacular -``` - -**Step 2: Update `settings.py`** - -You need to register the app and tell DRF to use Spectacular's schema generator. - -```python -INSTALLED_APPS = [ - # ... - 'drf_spectacular', -] - -REST_FRAMEWORK = { - # ... - 'DEFAULT_SCHEMA_CLASS': 'drf_spectacular.openapi.AutoSchema', -} - -# Optional: Add metadata for your Swagger UI -SPECTACULAR_SETTINGS = { - 'TITLE': 'Newsletter Maker API', - 'DESCRIPTION': 'API documentation for the newsletter maker app', - 'VERSION': '1.0.0', - 'SERVE_INCLUDE_SCHEMA': False, -} -``` - -**Step 3: Add the Swagger URLs** - -In your **`newsletter_maker/urls.py`** (the project root URL file), add the paths to serve the schema and the UI. - -```python -from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView - -urlpatterns = [ - # ... existing paths - path('api/schema/', SpectacularAPIView.as_view(), name='schema'), - # This is the actual Swagger page - path('api/docs/', SpectacularSwaggerView.as_view(url_name='schema'), name='swagger-ui'), -] -``` - -**How to Access It** - -Once you save the files and the server restarts: - -1. Navigate to `http://localhost:8000/api/docs/` in your browser. -2. You will see a live, interactive list of all your endpoints, including your **Tenants**, **Entities**, and **Review Queue**. -3. You can even click **"Try it out"** on any endpoint to send real requests directly from the documentation. - -Why use this over `show_urls`? - -- **Interactivity**: You can test endpoints with real data without needing a tool like Postman. -- **Model Schemas**: It shows exactly what JSON structure your serializers expect and return. -- **Documentation**: You can add human-readable descriptions to your endpoints using Python docstrings in your `api.py`. - -To enhance your Swagger UI with clear descriptions and examples, you use decorators provided by `drf-spectacular`. This allows you to document what each endpoint does and what the data should look like without changing your actual business logic. - -**Documenting ViewSet Actions** - -Because your `ViewSets` (like `TenantViewSet`) provide multiple actions (list, create, etc.) in one class, you use the `@extend_schema_view` decorator to target each action individually. - -**Example for your `TenantViewSet` in `api.py`:** - -```python -from drf_spectacular.utils import extend_schema, extend_schema_view, OpenApiExample - -@extend_schema_view( - list=extend_schema( - summary="List all tenants", - description="Returns a full list of all tenants registered in the system.", - ), - retrieve=extend_schema( - summary="Get tenant details", - description="Returns the detailed configuration and status for a single tenant.", - examples=[ - OpenApiExample( - 'Example Tenant Response', - value={ - "id": 1, - "name": "Acme Corp", - "slug": "acme-corp", - "status": "active" - } - ) - ] - ) -) -class TenantViewSet(viewsets.ModelViewSet): - # ... your existing code -``` - -**2. Adding Parameter Descriptions** - -For your nested URLs (like `/tenants//entities/`), you can explicitly describe what the `tenant_id` is for so it shows up in the Swagger "Parameters" section. - -```python -from drf_spectacular.utils import OpenApiParameter - -@extend_schema( - parameters=[ - OpenApiParameter( - name='tenant_id', - type=int, - location=OpenApiParameter.PATH, - description='The unique ID of the parent tenant' - ), - ], - description="Retrieve all entities belonging to a specific tenant." -) -def list(self, request, *args, **kwargs): - return super().list(request, *args, **kwargs) -``` - -**3. Using Docstrings for "Auto-Docs" ** - -If you want a simpler approach, `drf-spectacular` automatically picks up standard Python **docstrings** from your methods and uses them as descriptions in the Swagger UI. - -```python -def create(self, request, *args, **kwargs): - """ - Create a new newsletter source configuration. - - This endpoint validates the RSS/Atom feed URL before saving. - """ - return super().create(request, *args, **kwargs) -``` - -**4. Grouping with Tags** - -You can group your endpoints into logical sections (e.g., "Management", "Ingestion", "AI") using the `tags` parameter. This makes a long list of endpoints much easier to navigate in the browser. - -```python -@extend_schema(tags=['Tenant Management']) -class TenantViewSet(viewsets.ModelViewSet): - # ... -``` - -**Summary of Key Tools** - -- **`@extend_schema_view`**: Used on the class to document multiple actions at once. -- **`@extend_schema`**: Used on a single method for deep customization. -- **`OpenApiExample`**: Used to show users exactly what a sample JSON request or response looks like. - -To add custom validation that is both functional and documented in Swagger, you must override the `get_queryset` method or specific action methods in your `api.py` and then use `responses` in your `@extend_schema` decorator. - -**1. The Code: Custom 404 Validation** - -You can use `get_object_or_404` to ensure a tenant exists before allowing access to related data. In **DRF**, this automatically raises an `Http404` exception, which the framework translates into a clean JSON error response. - -**Example in `core/api.py`:** - -```python -from django.shortcuts import get_object_or_404 -from rest_framework import viewsets -from .models import Tenant, Entity - -class EntityViewSet(viewsets.ModelViewSet): - def get_queryset(self): - # Captures 'tenant_id' from the URL we defined earlier - tenant_id = self.kwargs.get("tenant_id") - - # This will raise a 404 if the tenant doesn't exist - tenant = get_object_or_404(Tenant, id=tenant_id) - - # If tenant exists, return only their entities - return Entity.objects.filter(tenant=tenant) -``` - -**2. The Documentation: Custom 404 Responses** - -To make this "Tenant Not Found" error visible in Swagger, you update your `@extend_schema` to include a **404 response**. - -```python -from drf_spectacular.utils import extend_schema, OpenApiResponse - -@extend_schema( - summary="List tenant entities", - responses={ - 200: EntitySerializer(many=True), - 404: OpenApiResponse( - description="Tenant not found", - # This shows the exact JSON the user will receive - examples=[ - OpenApiExample( - "Standard 404", - value={"detail": "No Tenant matches the given query."} - ) - ] - ), - } -) -def list(self, request, *args, **kwargs): - return super().list(request, *args, **kwargs) -``` - -**Why this is better than built-in validation:** - -- **Security**: It prevents users from "guessing" if a tenant exists by returning a 404 for non-existent tenants instead of an empty list `[]`. -- **Explicit Docs**: By defining `responses={404: ...}`, a new red "404" row appears in your Swagger UI, telling API consumers exactly what to expect if they use an invalid `tenant_id`. -- **Cleaner Logs**: Since `get_object_or_404` is a standard exception, your **Structlog** [turn 17] will capture it correctly without extra boilerplate code. - -Using the drf-standardized-errors package is the professional way to clean up your API's error responses and automatically document them in Swagger. It replaces Django REST Framework's default, sometimes inconsistent error formats with a single, structured JSON format. - -**1. Install and Configure** - -First, install the package with the OpenAPI support extension: - -```bash -docker exec -it newsletter-maker-django-1 pip install "drf-standardized-errors[openapi]" -``` - -Then, update your **`settings.py`** to tell DRF to use this new handler: - -```python -INSTALLED_APPS = [ - # ... - "drf_standardized_errors", -] - -REST_FRAMEWORK = { - # ... - "EXCEPTION_HANDLER": "drf_standardized_errors.handler.exception_handler", -} -``` - -**2. Automatic Documentation** - -Once registered, this package works with [drf-spectacular](https://drf-spectacular.readthedocs.io/) to automatically inject standardized error schemas (like 400, 401, and 404) into every endpoint in your Swagger UI. You no longer need to manually add `@extend_schema(responses={404: ...})` to every single view. - -**3. Standardized Error Format** - -Before this package, DRF might return different shapes for different errors. Now, every error (including those from your `get_object_or_404` call) will look like this: - -```json -{ - "errors": [ - { - "status": 404, - "code": "not_found", - "detail": "The requested resource was not found." - } - ] -} -``` - -This predictability is a massive benefit for frontend developers who need to write consistent error-handling logic in their code. - -**4. Customizing the Global Schema** - -If you find that standard 404 errors are cluttering every single endpoint in your docs (like your `list` views where a 404 is unlikely), you can configure drf-spectacular settings or the standardized errors settings to **hide certain error types** by default. - -## `api.py` - -This code is a sophisticated example of **Multi-Tenancy** and **Security**. Its primary goal is to ensure that users can only see and modify data belonging to a **Tenant** they own. - -Here is the breakdown of the two main strategies used: - -**The `TenantOwnedQuerysetMixin`** - -This is a reusable "plugin" (Mixin) that handles all the security logic for resources nested under a tenant (like `/tenants/1/entities/`). - -- **`get_tenant()`**: This is the gatekeeper. It looks at the URL for `tenant_id` and checks if the current user (`self.request.user`) actually owns that tenant. If they don't, it throws a `NotFound` error (404), preventing users from "snooping" on other people's tenant IDs. -- **`get_queryset()`**: Instead of showing every `Entity` in the database, it calls `filter(tenant=...)`. This ensures that even if a user manages to hit a "list" endpoint, they only see the data linked to the tenant validated in the step above. -- **`get_serializer_context()`**: It passes the `tenant` object into the Serializer. This is helpful if your serializer needs to do specific validation based on tenant settings. -- **`perform_create()`**: When you save a new object (like a new `Entity`), it **automatically attaches the tenant** to that object. The frontend doesn't need to send the tenant ID in the JSON body because the Mixin "injects" it from the URL. - -**The `TenantViewSet`** - -This manages the "Parent" objects. - -- **`select_related("user")`**: This is a performance optimization. It tells Django to join the User table in the SQL query so it doesn't have to make a separate database trip for every tenant in a list. -- **Ownership Check**: The `get_queryset` here filters by `user=self.request.user`, meaning when you visit `/api/v1/tenants/`, you only see *your* companies/tenants. - -**The ViewSets (The Implementation)** - -Notice how `TenantConfigViewSet`, `EntityViewSet`, etc., all inherit from `TenantOwnedQuerysetMixin`. - -- By simply adding that Mixin to the class definition, all 8 of those ViewSets instantly gain **automatic filtering**, **nested URL support**, and **security validation** without you having to rewrite that logic 8 times. -- **`select_related` everywhere**: The author is very careful with performance. For example, `ContentViewSet` joins `tenant` AND `entity`, which prevents the "N+1 query problem" (where the database gets hammered with tiny individual requests). - -**Special Case: `UserFeedbackViewSet`** - -Notice it overrides `perform_create`: - -```python -def perform_create(self, serializer): - serializer.save(tenant=self.get_tenant(), user=self.request.user) -``` - -This ensures that when feedback is submitted, the system automatically records **which tenant** it belongs to AND **which user** wrote it, pulled directly from the session/token. - -**Summary** - -This is **very high-quality Django code**. It uses the "Don't Repeat Yourself" (DRY) principle to enforce a strict security boundary. No matter which endpoint you hit, the system verifies your identity and your relationship to the data before showing a single row. - -Would you like to see how the **Serializers** use that `context["tenant"]` we added in the Mixin?** - -## GitHub Branch Checks - -GitHub required checks are configured in branch protection or rulesets, not in workflow YAML alone. After you push this, set these checks as required in GitHub: - -- Validate Branch Name -- Enforce PR Target -- Run test suite -- Run lint suite diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md deleted file mode 100644 index 7bfa2020..00000000 --- a/docs/IMPLEMENTATION.md +++ /dev/null @@ -1,496 +0,0 @@ -# Implementation Notes - -## Phase 1: MVP — Content Ingestion + Basic Surfacing - -**Goal:** Ingest content from RSS and Reddit, run it through a classification → relevance → summarization pipeline, and surface the most relevant items in a dashboard where editors can upvote/downvote content. - -### Work Packages - -Phase 1 is organized into seven work packages that build on each other. Each produces a testable, runnable increment. - ---- - -### WP1: Project Scaffold + Docker Compose - -Set up the Django project, Docker infrastructure, and development environment. - -**Deliverables:** - -- Django project (`newsletter_maker/`) with DRF installed -- `docker-compose.yml` with all services: - - | Service | Image | Purpose | - |---------|-------|---------| - | `django` | Custom Dockerfile (gunicorn) | API server | - | `celery-worker` | Same image, different entrypoint | Background task execution | - | `celery-beat` | Same image, different entrypoint | Scheduled task triggers | - | `postgres` | `postgres:16` | Relational data | - | `redis` | `redis:7-alpine` | Celery broker + cache | - | `qdrant` | `qdrant/qdrant:latest` | Vector storage | - | `nginx` | `nginx:alpine` | Reverse proxy | - -- `.env.example` with all required environment variables: - - `DATABASE_URL`, `REDIS_URL`, `QDRANT_URL` - - `OPENROUTER_API_KEY` - - `EMBEDDING_PROVIDER`, `EMBEDDING_MODEL` - - `OLLAMA_URL` for local Ollama embeddings - - `OPENROUTER_API_BASE` for hosted OpenAI-compatible embeddings APIs - - `SECRET_KEY`, `DEBUG`, `ALLOWED_HOSTS` -- Health check endpoints: `GET /healthz/` (system health), `GET /readyz/` (DB + Qdrant reachable) -- `justfile` with commands: `dev` (docker compose up), `test`, `migrate`, `seed`, `shell` -- `structlog` configured for JSON logging to stdout - -**Definition of done:** `just dev` brings up all services, `/healthz/` returns 200, Django admin is accessible. - -### WP2: Data Models - -Core Django models for multi-tenant content management. - -**Models:** - -```python -class Tenant(models.Model): - name = models.CharField(max_length=255) # Newsletter name - user = models.ForeignKey(User, on_delete=models.CASCADE) - topic_description = models.TextField() # "Platform engineering and DevOps" - content_retention_days = models.IntegerField(default=365) - created_at = models.DateTimeField(auto_now_add=True) - -class TenantConfig(models.Model): - tenant = models.OneToOneField(Tenant, on_delete=models.CASCADE) - upvote_authority_weight = models.FloatField(default=0.1) - downvote_authority_weight = models.FloatField(default=-0.05) - authority_decay_rate = models.FloatField(default=0.95) # Monthly multiplier - -class Entity(models.Model): - tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE) - name = models.CharField(max_length=255) - type = models.CharField(choices=ENTITY_TYPE_CHOICES) # individual, vendor, organization - description = models.TextField(blank=True) - authority_score = models.FloatField(default=0.5) - website_url = models.URLField(blank=True) - github_url = models.URLField(blank=True) - linkedin_url = models.URLField(blank=True) - bluesky_handle = models.CharField(max_length=255, blank=True) - mastodon_handle = models.CharField(max_length=255, blank=True) - twitter_handle = models.CharField(max_length=255, blank=True) - created_at = models.DateTimeField(auto_now_add=True) - - class Meta: - unique_together = ('tenant', 'name') - -class Content(models.Model): - tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE) - url = models.URLField() - title = models.CharField(max_length=512) - author = models.CharField(max_length=255, blank=True) - entity = models.ForeignKey(Entity, null=True, blank=True, on_delete=models.SET_NULL) - source_plugin = models.CharField(max_length=64) # "rss", "reddit" - content_type = models.CharField(max_length=64, blank=True) # Set by Classification skill - published_date = models.DateTimeField() - ingested_at = models.DateTimeField(auto_now_add=True) - content_text = models.TextField() - relevance_score = models.FloatField(null=True) - is_active = models.BooleanField(default=True) # Soft delete for retention - - class Meta: - indexes = [ - models.Index(fields=['tenant', '-published_date']), - models.Index(fields=['tenant', '-relevance_score']), - models.Index(fields=['url']), # Dedup on ingest - ] - -class SkillResult(models.Model): - content = models.ForeignKey(Content, on_delete=models.CASCADE, related_name='skill_results') - tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE) - skill_name = models.CharField(max_length=64) - status = models.CharField(choices=STATUS_CHOICES) # pending, running, completed, failed - result_data = models.JSONField(null=True) - error_message = models.TextField(blank=True) - model_used = models.CharField(max_length=64) - latency_ms = models.IntegerField(null=True) - confidence = models.FloatField(null=True) - created_at = models.DateTimeField(auto_now_add=True) - superseded_by = models.ForeignKey('self', null=True, blank=True, on_delete=models.SET_NULL) - - class Meta: - indexes = [ - models.Index(fields=['content', 'skill_name']), - models.Index(fields=['tenant', 'created_at']), - ] - -class UserFeedback(models.Model): - content = models.ForeignKey(Content, on_delete=models.CASCADE) - tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE) - user = models.ForeignKey(User, on_delete=models.CASCADE) - feedback_type = models.CharField(choices=[('upvote', 'Upvote'), ('downvote', 'Downvote')]) - created_at = models.DateTimeField(auto_now_add=True) - - class Meta: - unique_together = ('content', 'user') # One vote per user per item - -class IngestionRun(models.Model): - tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE) - plugin_name = models.CharField(max_length=64) - started_at = models.DateTimeField(auto_now_add=True) - completed_at = models.DateTimeField(null=True) - status = models.CharField(choices=RUN_STATUS_CHOICES) # running, success, failed - items_fetched = models.IntegerField(default=0) - items_ingested = models.IntegerField(default=0) - error_message = models.TextField(blank=True) - - class Meta: - indexes = [ - models.Index(fields=['tenant', 'plugin_name', '-started_at']), - ] - -class ReviewQueue(models.Model): - tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE) - content = models.ForeignKey(Content, on_delete=models.CASCADE) - reason = models.CharField(max_length=64) # low_confidence_classification, borderline_relevance - confidence = models.FloatField() - created_at = models.DateTimeField(auto_now_add=True) - resolved = models.BooleanField(default=False) - resolution = models.CharField(max_length=64, blank=True) # human_approved, human_rejected -``` - -**DRF serializers and viewsets** for all models, scoped by `request.user`'s tenant. - -**Definition of done:** Models migrated, Django admin registered, API endpoints return JSON for all models, tenant scoping enforced. - -### WP3: Data Source Plugins (RSS + Reddit) - -Implement the plugin interface and the two Phase 1 plugins. - -**Plugin interface:** - -```python -class SourcePlugin(ABC): - @abstractmethod - def fetch_new_content(self, since: datetime) -> list[ContentItem]: ... - - @abstractmethod - def health_check(self) -> bool: ... -``` - -**RSS plugin:** - -- Uses `feedparser` to fetch configured feed URLs -- Stores seen entry GUIDs to avoid re-ingesting duplicates -- Extracts: URL, title, author, published date, summary/excerpt -- Links to Entity if feed URL matches a tracked entity's website -- Scheduled via Celery Beat: every 6 hours - -**Reddit plugin:** - -- Uses PRAW (Python Reddit API Wrapper) -- Monitors configured subreddits via `.new()` and `.hot()` -- Extracts: title, selftext, URL, score, subreddit, author -- Tracks upvote count as a quality signal -- Does **not** link Reddit users to Entity profiles -- Scheduled via Celery Beat: every 6 hours - -**Plugin configuration** stored in a `SourceConfig` model: - -```python -class SourceConfig(models.Model): - tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE) - plugin_name = models.CharField(max_length=64) # "rss", "reddit" - config = models.JSONField() # Plugin-specific: {"feed_url": "..."} or {"subreddit": "..."} - is_active = models.BooleanField(default=True) - last_fetched_at = models.DateTimeField(null=True) -``` - -**Celery tasks:** - -- `run_ingestion(tenant_id, plugin_name)` — fetches new content, creates `Content` records, logs `IngestionRun` -- `run_all_ingestions()` — Beat-scheduled task that triggers ingestion for all active source configs - -**Natural next steps:** - -- Add a small management command or just target to trigger run_all_ingestions and run_ingestion manually during development. -- Add source health reporting so /admin/health or a tenant API endpoint can surface RSS/Reddit plugin status from health_check(). -- Tighten the ingestion contract by storing plugin-specific metadata such as Reddit score/subreddit or RSS entry IDs if you want better deduplication and debugging. - -**Definition of done:** RSS plugin ingests from a real feed. Reddit plugin ingests from a real subreddit. `IngestionRun` records log success/failure. Health checks return correct status. - -What is not built yet is the full done state from the plan, especially live feed/subreddit wiring in a running stack, source-health UI/endpoints, and richer ingestion metadata/dedup strategy. - -### WP4: Embeddings + Qdrant Integration - -Compute embeddings for all ingested content and store them in Qdrant for similarity search. - -**Embedding backend:** Configurable via `EMBEDDING_PROVIDER` plus `EMBEDDING_MODEL`. - -- `sentence-transformers`: local Hugging Face / SentenceTransformers model loading -- `ollama`: local model served over Ollama's embedding API -- `openrouter`: hosted embeddings through the OpenRouter `/embeddings` API - -Examples: - -- `EMBEDDING_PROVIDER=sentence-transformers`, `EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2` -- `EMBEDDING_PROVIDER=ollama`, `EMBEDDING_MODEL=nomic-embed-text` -- `EMBEDDING_PROVIDER=ollama`, `EMBEDDING_MODEL=qwen3-embedding-8b` -- `EMBEDDING_PROVIDER=openrouter`, `EMBEDDING_MODEL=openai/text-embedding-3-small` - -**Qdrant setup:** - -- One collection per tenant (named `tenant_{id}_content`) -- HNSW index for fast similarity search -- Each point stores: vector, plus payload metadata (content ID, URL, title, published date, source plugin) - -**Integration flow:** - -1. Plugin fetches article → `Content` record created in Postgres -2. Post-save signal (or inline in ingestion task) computes embedding -3. The configured embedding provider returns a vector for the content text -4. Embedding + metadata upserted into tenant's Qdrant collection -5. `Content.embedding_id` stores the Qdrant point ID for later retrieval - -**Reference corpus seeding:** - -- Each tenant's Qdrant collection is initialized with embeddings from reference articles that define "what's relevant" for this newsletter -- Seed script pre-populates with ~50-100 sample articles per tenant -- These reference embeddings serve as the relevance baseline for the scoring skill - -**Utility functions:** - -- `embed_text(text: str) -> list[float]` — compute embedding for a text string -- `upsert_content_embedding(content: Content)` — embed and store in Qdrant -- `search_similar(tenant_id: int, query_vector: list[float], limit: int) -> list[ScoredPoint]` — find similar content -- `get_reference_similarity(tenant_id: int, vector: list[float]) -> float` — average similarity against reference corpus - -**Operational usage:** - -- `just embed-all` — backfill embeddings for all content rows -- `just embed-tenant ` — backfill one tenant's content -- `python3 manage.py sync_embeddings --content-id ` — re-embed one record - -**Definition of done:** Every ingested content item has an embedding in Qdrant. `search_similar` returns semantically related articles. Reference corpus is seeded for test tenant. - -- Run `just embed-smoke` to confirm Django can talk to Ollama. -- If that works, run `just embed-all` or `just embed-tenant ` to backfill real content. - -Natural next steps: - -- add a DRF endpoint or admin action that returns similar content for a given item using search_similar_content -- expand the seeded reference corpus from the current minimal set into a more realistic tenant baseline -- wire WP5 relevance scoring to get_reference_similarity and the seeded reference items - -### WP5: AI Skills + LangGraph Pipeline - -Implement the three Phase 1 skills and wire them into a LangGraph orchestrator. - -**Skill format:** Each skill lives in a `skills/{skill_name}/` directory following Claude-style progressive disclosure: - -```bash -skills/ -├── content_classification/ -│ ├── SKILL.md # YAML frontmatter (Level 1) + SOP instructions (Level 2) -│ └── references/ # Category definitions, examples (Level 3) -├── relevance_scoring/ -│ ├── SKILL.md -│ └── references/ # Scoring rubric, example judgments -└── summarization/ - ├── SKILL.md - └── references/ # Style guide, example summaries -``` - -**Skill implementations (Python wrappers around the prompt + model call):** - -**1. Content Classification** (Llama 3.1 70B via OpenRouter) - -- Input: `{title, content_text, url}` -- Output: `{content_type: str, confidence: float}` -- Categories: technical_article, tutorial, opinion, product_announcement, event, release_notes, other -- If `confidence < 0.6`: add to `ReviewQueue` with reason `low_confidence_classification` - -**2. Relevance Scoring** (Embeddings primary, Qwen 2.5 72B for borderline) - -- Input: `{content_embedding, tenant_id}` -- Step 1: Compute cosine similarity against tenant's reference corpus in Qdrant -- Step 2: If similarity > 0.85 → score = similarity (no LLM call). If similarity < 0.5 → score = similarity (no LLM call). If 0.5 - 0.85 → call Qwen for nuanced judgment with explanation. -- Output: `{relevance_score: float, explanation: str, used_llm: bool}` -- If `0.4 < relevance_score < 0.7`: add to `ReviewQueue` with reason `borderline_relevance` - -**3. Summarization** (Gemma 3 27B via OpenRouter) - -- Input: `{title, content_text, newsletter_topic}` -- Output: `{summary: str}` (2-3 sentences, newsletter-ready) -- Only runs on content with `relevance_score >= 0.7` - -**LangGraph pipeline:** - -```python -from langgraph.graph import StateGraph, END - -class PipelineState(TypedDict): - content_id: int - tenant_id: int - classification: dict | None - relevance: dict | None - summary: dict | None - status: str # "processing", "completed", "archived", "review" - -def build_ingestion_graph(): - graph = StateGraph(PipelineState) - graph.add_node("classify", classify_node) - graph.add_node("score_relevance", relevance_node) - graph.add_node("summarize", summarize_node) - graph.add_node("archive", archive_node) - graph.add_node("queue_review", queue_review_node) - - graph.set_entry_point("classify") - graph.add_edge("classify", "score_relevance") - graph.add_conditional_edges("score_relevance", route_by_relevance, { - "relevant": "summarize", - "borderline": "queue_review", - "irrelevant": "archive", - }) - graph.add_edge("summarize", END) - graph.add_edge("archive", END) - graph.add_edge("queue_review", END) - - return graph.compile(checkpointer=redis_checkpointer) -``` - -**Routing logic:** - -- `relevance_score >= 0.7` → summarize → surface in dashboard -- `0.4 <= relevance_score < 0.7` → queue for human review (surfaced with "Low confidence" flag) -- `relevance_score < 0.4` → archive (soft-delete, `is_active = False`) - -**Error handling:** - -- Each node wrapped in try/except; failures logged as `SkillResult` with `status=failed` -- Max 2 retries per node before marking as failed and moving to review queue -- Circuit breaker: after 5 consecutive failures on a skill, pause pipeline and alert via log - -**Celery integration:** - -- `process_content(content_id)` — Celery task that runs the LangGraph pipeline for one content item -- Called automatically after ingestion creates a new `Content` record -- State checkpointed in Redis — if worker dies mid-pipeline, resumes from last completed node - -**Definition of done:** New content ingested via RSS/Reddit automatically flows through classify → score → summarize. Results persisted as `SkillResult` records. Borderline items appear in review queue. Irrelevant items archived. - ---- - -### WP6: React Frontend (Minimal Dashboard) - -A focused UI for reviewing and acting on surfaced content. - -**Pages:** - -1. **Content Dashboard** (`/`) - - List of surfaced content for current period, sorted by relevance score - - Each item shows: title, source, published date, relevance score, content type badge - - Upvote/downvote buttons on each item - - Click to expand → full detail view with skill action bar - - Filter by: content type, date range, source plugin - - "Pending review" tab showing items from the review queue - -2. **Content Detail** (expanded view or `/content/{id}`) - - Full article metadata - - `` component with buttons: - - **Summarize** — triggers summarization skill, shows result inline - - **Explain Relevance** — triggers relevance scoring skill, shows score + explanation - - **Find Related** — queries Qdrant for similar embeddings, shows related articles - - Each skill result renders in a collapsible section with Copy and Regenerate actions - - Persisted results shown immediately if already generated; "Regenerate" creates a new result - -3. **Entity Management** (`/entities`) - - CRUD for entities (name, type, description, platform handles) - - Simple form — no automated discovery or linking in Phase 1 - -4. **Ingestion Health** (`/admin/health`) - - Status of each data source plugin: last fetch, items count, error status - - Color-coded: green (healthy), yellow (degraded), red (failing) - -5. **Source Configuration** (`/admin/sources`) - - Add/edit/disable RSS feeds and Reddit subreddits - - Per-source health status - -**API integration:** - -- All data fetched via DRF endpoints -- Skill invocations: `POST /api/v1/tenants/{tenant_id}/contents/{id}/skills/{skill_name}/` - - Returns `202 Accepted` for async skills (summarization) - - Frontend polls `GET /api/v1/tenants/{tenant_id}/skill-results/{id}/` until `status=completed` - - Returns `200` with result for fast skills (find related via Qdrant) -- Feedback: `POST /api/v1/tenants/{tenant_id}/feedback/` with `{content: , feedback_type: "upvote"|"downvote"}` - -**Definition of done:** User can view dashboard, expand articles, trigger skills, see results inline, upvote/downvote content, manage entities and sources. - ---- - -### WP7: Seed Script + Demo Data - -Make the project immediately demo-able without weeks of data accumulation. - -**`just seed` command runs a management command that:** - -1. Creates a demo tenant ("Platform Engineering Weekly") with topic description -2. Creates ~15 entities: - - 8 individuals (Kelsey Hightower, Charity Majors, etc.) - - 5 vendors (HashiCorp, Datadog, Grafana Labs, etc.) - - 2 organizations (CNCF, Linux Foundation) -3. Creates ~200 content items spanning 30 days: - - ~150 from RSS (mix of relevant and irrelevant) - - ~50 from Reddit (r/kubernetes, r/devops posts with realistic scores) -4. Embeds all content and seeds Qdrant with reference corpus (~50 curated articles) -5. Runs the LangGraph pipeline on all content to populate: - - Classification results - - Relevance scores - - Summaries for relevant items - - Review queue entries for borderline items -6. Adds sample user feedback (~30 upvotes, ~15 downvotes) to demonstrate feedback state -7. Creates sample `IngestionRun` records showing healthy ingestion history - -**Definition of done:** `just seed` produces a fully populated dashboard with realistic data, scored and summarized content, and a non-empty review queue. A new user can explore the full UI immediately. - ---- - -### Work Package Dependencies - -```bash -WP1 (Scaffold) - └─► WP2 (Models) - ├─► WP3 (Plugins) - │ └─► WP4 (Embeddings) - │ └─► WP5 (Skills + Pipeline) - │ └─► WP7 (Seed Script) - └─► WP6 (Frontend) - └─► WP7 (Seed Script) -``` - -WP6 (Frontend) can begin in parallel with WP3-WP5 once models and API endpoints exist — it can develop against mock data initially and integrate with real endpoints as they land. - ---- - -### Phase 1 Cost Summary - -| Category | Monthly Cost | -|----------|-------------| -| VPS hosting (4GB+ RAM) | ~$30 | -| OpenRouter LLM calls (1 tenant, ~2K articles) | ~$2.30 | -| Embeddings (local) | $0 | -| Qdrant (self-hosted) | $0 | -| PostgreSQL + Redis (self-hosted) | $0 | -| **Total** | **~$32/month** | - -### What's Explicitly Out of Scope - -- Newsletter email ingestion (Resend) → Phase 2 -- Bluesky / Mastodon / LinkedIn plugins → Phase 2+ -- Deduplication skill → Phase 2 -- Entity extraction skill → Phase 2 -- Theme detection skill → Phase 3 -- Authority scoring in relevance ranking → Phase 2 (scores exist but aren't used in ranking yet) -- Topic centroid feedback loop → Phase 2 -- Few-shot training from feedback → Phase 3 -- Multi-step skill chaining UI → Phase 3 -- Saved workflow templates → Phase 4 -- Prometheus / Grafana / external observability → Phase 3+ -- Kubernetes deployment → Phase 2+ -- CI/CD beyond basic GitHub Actions → Phase 2+ diff --git a/docs/PLANNING.md b/docs/PLANNING.md deleted file mode 100644 index 1b26a226..00000000 --- a/docs/PLANNING.md +++ /dev/null @@ -1,656 +0,0 @@ -# Planning Discussion - -## Project Goals - -Help curate content for regular technically-oriented newsletters. The tool is multi-tenant — it supports multiple newsletters on different topics, maintained by different users. This lays the groundwork for potential SaaS use or adoption by a team at a newsletter company where different editors maintain different newsletters. - -The UI must be designed for non-technical end users. This is a product, not a developer tool. - -**Composability is the core philosophy.** Every capability in the system is a self-contained unit with clean inputs/outputs that can be invoked independently or chained together. The plugin architecture already points this direction, but the AI pipeline itself needs to be decomposed into composable units too — not just a monolithic LangGraph DAG. - -**Model-agnostic Skills, not vendor lock-in.** The architecture uses Claude-style Skills as the module format — each AI capability (content classification, summarization, relevance scoring, entity extraction) is a standalone, documented "Skill" with a clear interface that a non-technical user could understand and invoke. However, the Skills format is model-agnostic: GPT models can consume Claude Skills directly, and local models via Ollama may be able to as well. This lets us use the right model for each task — Claude Sonnet for high-value generation, cheaper/local models for commodity tasks — without changing the Skill definitions. - -**Non-technical users are half the job.** The UI needs to be polished, intuitive, and designed for editors who don't know what a vector database is. But beyond the UI, the system design itself should be legible — a non-technical person should be able to understand what the system does by reading the skill/tool descriptions. - -**Production agents, not demos.** The background ingestion pipeline needs proper error handling, retry logic, health monitoring, and clear escalation paths (e.g., "this newsletter email couldn't be parsed, flagging for human review" rather than silently failing). - -**Systems of record integration.** The plugin architecture allows easy integration with HubSpot, Google Drive, and Slack. - -### What the Tool Should Do - -- Surface the most important articles, events, and ideas relevant to a newsletter's topic since its last edition was published -- Suggest themes and ideas for original content (e.g. "Our Insight" segments, blog articles) -- Over time, build and maintain a database of people, companies, and organizations worth tracking -- Ingest new content from those entities and make sense of it for the newsletter writer -- Let users give feedback (upvote/downvote) on surfaced content and entities to train the relevance model over time - -### Hard Problems - -- **Content deduplication:** The same announcement gets covered by 15 blogs. The tool should recognize that and surface the best version, not all 15. -- **Trend velocity:** Not just "what exists" but "what is gaining traction *right now*" — a signal that matters for weekly publishing cadence. -- **Source diversity:** A good newsletter doesn't link to the same 3 sites every week. The tool should help balance sources. -- **Freshness decay:** Content from 6 days ago is stale for a weekly newsletter. The ranking model needs a time-decay factor. -- **Signal vs. noise in social media:** Distinguishing genuine technical discussion from hype. Rather than trying to build an automated marketing-vs-technical classifier (too unreliable), we rely on authority scoring, user feedback, and source quality signals to filter noise organically. -- **Graceful failure and escalation:** Production AI systems need to handle edge cases — unparseable newsletters, ambiguous entity matches, API outages — and know when to flag for human review rather than silently failing or producing garbage. - -### Differentiators vs. Existing Tools - -Existing tools like Feedly, UpContent, and ContentStudio handle parts of this problem. Our unique value comes from combining several things none of them do: - -1. **Authority scoring from newsletter cross-referencing.** By ingesting peer newsletters, we build an authority model based on who real editors actually link to — a signal no existing tool provides. This is the strongest differentiator. -2. **Per-user relevance training via feedback loops.** The upvote/downvote system trains a personalized relevance model for each newsletter editor. Over time, the tool learns what *you* consider valuable. Feedly's Leo AI learns preferences, but not in a multi-tenant context where each editor has a separate model. -3. **Unified entity model.** No existing tool links a person's blog, LinkedIn, Bluesky, GitHub, and conference talks into a single profile with an authority score. This gives a holistic view of who matters in the space. -4. **Competitive intelligence.** "These 3 peer newsletters all covered topic X this week, but you haven't." This is a natural output of newsletter ingestion that no curation tool provides. -5. **Historical trend analysis.** "This topic has been growing across sources over the last 4 weeks." Most tools show what's trending *now*; we can show trajectories because we retain content indefinitely. -6. **Topic-tuned relevance.** The vector DB is seeded with domain-specific reference content, so relevance scoring is tuned to the newsletter's niche rather than being generic engagement-based filtering. - -## Architecture Overview - -### Core Pipeline - -The system has three logical stages: - -1. **Entity Management** — Build and update databases of people, companies/vendors, and organizations to track. Link their various accounts (LinkedIn, personal site, social media) into unified profiles. Users can upvote/downvote entities to adjust their authority scores. -2. **Content Ingestion** — Pull new content from tracked entities and monitored channels (RSS, social media, newsletters, Reddit). Runs continuously via scheduled background jobs. -3. **Analysis & Surfacing** — Rank, deduplicate, and present the most relevant content for the current newsletter edition. Users upvote/downvote surfaced content to train their personal relevance model. - -### User Workflow - -The backend is an always-running server with periodic background jobs that scrape and ingest content. The user opens the tool when they have work to do: - -- **Weekly (before writing):** Review the dashboard of surfaced content for the current period. Upvote/downvote items. Select content for inclusion in the newsletter. -- **Periodically:** Enrich entity data — link social accounts to profiles, review suggested entity matches, add new entities discovered from newsletter ingestion. -- **Occasionally:** Configure sources, adjust settings, review ingestion health. - -No push notifications. The system accumulates and ranks content between user sessions. - -### Tech Stack - -- **Backend:** Django + Django REST Framework -- **AI Pipeline:** Composable Skills architecture (model-agnostic, Claude-style progressive disclosure format) orchestrated by LangGraph. Multi-model: Claude Sonnet for high-value generation, local models (Ollama) or Claude Haiku for commodity tasks, embeddings for similarity operations. -- **Vector DB:** Self-hosted Qdrant for semantic search over ingested content -- **Relational DB:** PostgreSQL for entities, relationships, ingestion history, authority scores, user feedback, tenant config -- **Background Jobs:** Celery + Redis for scheduled scraping, ingestion runs, and periodic maintenance (content purging) -- **Email Intake:** Resend API for receiving newsletter subscription emails -- **Frontend:** React, designed for non-technical users -- **Design:** Pluggable architecture with clear API contracts for data source plugins - -### Multi-Tenant Architecture - -Each tenant represents a newsletter. Tenants have: - -- Their own set of tracked entities (entities are **not** shared across tenants — the same person may have separate records in different tenants with different authority scores, since someone who is high-authority for a DevOps newsletter may be low-authority for a frontend newsletter) -- Their own relevance model, trained by that editor's upvote/downvote feedback -- Their own newsletter ingestion subscriptions -- Their own content surfacing dashboard, scoped to their topic -- Configurable content retention/purge window (default: 1 year) - -> **Implementation note:** Django's built-in auth + a tenant model with ForeignKey relationships on all content/entity tables is sufficient to start. Don't reach for a multi-tenant library like `django-tenants` (which uses Postgres schemas) unless you actually need schema-level isolation. Simple row-level tenant scoping works fine until you have hundreds of tenants. - -### Content Retention - -Content is stored indefinitely by default, which enables historical trend analysis and authority scoring over time. A configurable per-tenant purge window (e.g. 1 year) runs as a periodic Celery task to clean up old content. The purge should soft-delete or archive rather than hard-delete, so authority score history isn't lost. - -## Entity Model - -### Three Types of Content Producers - -| Type | Content Sources | Social Media Value | -|------|----------------|-------------------| -| **Individuals** | Personal sites, vendor publications, syndication platforms (Substack, DZone, dev.to, Medium) | High — technical discussion, opinions | -| **Vendors** | Company blogs, docs, changelogs, release notes | Low — mostly marketing | -| **Organizations** | Websites of industry groups, standards bodies, NGOs, government agencies | Medium — official announcements | - -### Unified Profiles (Entity Resolution) - -Each entity should have a unified profile that links together all their accounts: LinkedIn, personal website, GitHub, social media handles, etc. - -**This is one of the hardest parts of the project.** Cross-platform identity resolution is a genuinely difficult problem. Consider: - -- Name matching is unreliable (John Smith problem, name variations, pseudonyms) -- Not everyone uses consistent handles across platforms -- Automated linking will produce false positives - -**Practical approach:** Don't try to fully automate this. Start with manual profile creation (the user adds entities and links their accounts). Over time, add *suggestions* — "this Bluesky account might belong to this person based on bio similarity and shared links" — but require human confirmation. The LLM pipeline could help here by comparing bios and writing samples. - -For the MVP, a simple entity form with fields for each platform handle is enough. Automated entity discovery is a later feature. - -### Authority Scoring - -We need a way to rank content producers, analogous to Ahrefs' Domain Authority but tuned for newsletter relevance. - -**The highest quality signal** for adding someone to the system is when they're featured in an existing industry newsletter. That's a human-curated endorsement of relevance. - -A composite "Authority Score" could weight several factors: - -- **Newsletter mention count:** How often does this person/company appear in newsletters we track? (Strongest signal) -- **Newsletter mention recency:** Recent mentions matter more than old ones -- **Content engagement:** Social shares, Reddit upvotes on their content -- **Publishing frequency:** Active contributors vs. one-off authors -- **Domain expertise signals:** Job title, GitHub contributions, conference talks (harder to automate) - -Don't try to build this scoring system perfectly upfront. Start with newsletter-mention-count as the primary signal and add dimensions incrementally. User upvote/downvote feedback on entities provides an additional per-tenant adjustment to authority scores. - -## Data Source Plugins - -### Plugin Architecture - -The pluggable design is a good instinct. Each data source should implement a common interface: - -```python -class SourcePlugin: - def fetch_new_content(since: datetime) -list[ContentItem] - def get_entity_profile(entity_id) -EntityProfile | None - def health_check() -bool -``` - -Each plugin handles its own auth, rate limiting, and API-specific concerns. The core system just calls the interface. A plugin registry and configuration system lets users enable/disable sources and provide API keys. - -### Newsletter Ingestion - -Subscribing to and parsing existing industry newsletters is the most important data source — they provide both content signals and authority signals. The content from newsletters is "dated" by the time you process it, so it's not directly useful for your own "what's trending" snapshot. Its real value is as an authority signal — who and what are peer newsletters linking to? - -Receive newsletters as email via the Resend Inbound API, then use Claude to extract structured link/article data. Use the extracted data for two things: (a) adding mentioned people/companies to the entity database, and (b) tracking what topics peer newsletters are covering. - -#### Email Intake via Resend - -Use [Resend's Inbound Email](https://resend.com/docs/dashboard/webhooks/introduction) feature: - -1. Configure a subdomain (e.g. `inbox.newslettermaker.dev`) with Resend's MX records -2. Resend receives emails to any address at that subdomain and forwards them to our webhook as structured JSON (sender, subject, HTML body, text body, attachments) -3. Each tenant gets a unique intake address (e.g. `tenant-xyz@inbox.newslettermaker.dev`). The user subscribes to peer newsletters using this address. -4. On webhook receipt: store the raw email, queue an LLM extraction job, parse out article links/authors/topics - -#### Resend Inbound Limits & Handling - -- **Total Volume Limit:** 3,000 emails per month (combined sending and receiving). -- **Total Size Limit:** Inbound emails cannot exceed 40MB after Base64 encoding. Base64 encoding typically increases file size by ~33%, so a 30MB raw email may exceed the limit once encoded. -- **Rejection:** Oversized emails are rejected by the mail server; the sender receives a bounce/NDR message. Resend does **not** strip attachments to fit — the entire message is rejected. -- **Attachments:** Resend's inbound webhooks provide attachment **metadata** and a temporary download URL via the [Attachments API](https://resend.com/docs/api-reference/emails/list-received-email-attachments), rather than including attachment data in the webhook POST payload. Most standard file types are supported; executable files (`.exe`, etc.) are [blocked for security](https://resend.com/docs/dashboard/emails/attachments). Inline images (CID) are treated as attachments and count toward the 40MB limit. -- **Implication for this project:** Newsletter emails are typically text/HTML with embedded images, well under 40MB. The 3,000/month combined limit is the main constraint — at scale, this may require upgrading Resend plans or batching sends. For the MVP with a handful of tenants tracking 10-20 newsletters each, the free tier is sufficient. - -#### Newsletter Subscription Confirmation - -Most newsletter platforms send a confirmation/double-opt-in email after signup. We need to handle this: - -- **Detection:** When a new email arrives from an unrecognized sender, the LLM extraction step should classify it as either "confirmation request" or "newsletter content." Confirmation emails have recognizable patterns (single CTA button, "confirm your subscription" language). -- **Workflow:** Surface unconfirmed subscriptions in the UI with a "pending confirmation" status. Show the user the confirmation link extracted from the email so they can click it manually. Fully automating the click is possible but risks triggering anti-bot measures from newsletter platforms. -- **After confirmation:** Mark the subscription as active. Future emails from that sender go straight to the LLM extraction pipeline. - -### RSS Feeds - -Track websites/blogs of followed entities and scrape new content as it appears. - -Straightforward to implement. Use `feedparser` in Python. Run on a schedule (e.g. every 6 hours). Store seen entry IDs to avoid duplicates. This should be one of the first plugins built — it's simple, reliable, and high value. - -### Reddit - -A special case — we're watching subreddits for trends, not tracking individual authors. - -PRAW makes this easy: -- Monitor specific subreddits (e.g. `r/platformengineering`, `r/devops`, `r/kubernetes`) -- Use `.new()` for recent posts, `.hot()` for trending -- Track upvote velocity as a quality signal -- `.stream()` for real-time monitoring - -**Don't try to link Reddit users to entity profiles** — as noted in the original plan, it doesn't matter. Reddit's value is trend detection and community sentiment, not individual tracking. Surface popular questions, recurring pain points, and well-received discussions. - -### Social Media Platforms - -Watch the output of tracked entities on Bluesky and Mastodon. - -**What about tracking for hashtags on Bluesky and Mastodon?** - -- **Bluesky:** Open AT Protocol. Full API access. No cost. This is the easiest platform to integrate. **Start here for social media.** - -- **Mastodon:** Open ActivityPub protocol. Per-instance APIs, all free. Also easy. **Second priority.** - -### LinkedIn with API Details - -LinkedIn for enhancing entities, and adding suggestions for articles + their related entities - -- **Profile Data:** Can access public profile info (name, headline, location, experience) and connections, but only for authenticated users or approved apps. -- **Keyword Search for Articles:** Not available through official API. Workaround: Google Search with `site:linkedin.com/posts "keyword"` and time filtering. - -### Cost Considerations - -Agree with the principle of keeping third-party API costs minimal and preferring pay-as-you-go. - -**Estimated baseline costs for external services:** - -- Bluesky API: Free -- Mastodon API: Free -- Reddit API: Free (within rate limits) -- RSS parsing: Free (self-hosted) -- LLM calls (for content analysis, summarization, classification): Variable — see Multi-Model Strategy in the AI Pipeline section and [VENDOR.md](VENDOR.md) for per-skill model selection and pricing. - -**Development API strategy: OpenRouter.** During development, all LLM calls go through [OpenRouter](https://openrouter.ai/) as a unified API gateway. This provides access to all target models (Llama, Command R+, Gemma, DeepSeek, Qwen) through a single API key and billing account, with pay-as-you-go pricing. No need to manage separate vendor accounts during development. The skill interface is endpoint-agnostic — when transitioning to self-hosted Ollama for production, only the base URL and model identifier change in config. - -**Estimated development costs via OpenRouter** (based on ~2,000 articles/month ingestion for one tenant): - -| Skill | Model | Est. monthly calls | Est. cost | -|-------|-------|--------------------|-----------| -| Content Classification | Llama 3.1 70B | 2,000 | ~$1.60 | -| Relevance Scoring | Embeddings primary, Command R+ for borderline | ~200 LLM calls | ~$2.50 | -| Deduplication | Embeddings primary, Command R+ for borderline | ~100 LLM calls | ~$1.25 | -| Summarization | Gemma 3 27B | ~800 (relevant items) | ~$0.20 | -| Theme Detection | DeepSeek V3.2 | ~30 (daily) | ~$0.20 | -| Newsletter Extraction | Qwen 2.5 72B | ~60 (newsletters) | ~$0.03 | -| Entity Extraction | Qwen 2.5 72B | ~200 | ~$0.10 | -| **Total** | | | **~$6/month** | - -This is dramatically lower than the original $20-30/month estimate because every skill now uses a cost-optimized model rather than defaulting to Claude Sonnet. The embeddings-first approach for relevance scoring and deduplication is critical — without it, Command R+ at $10/M output tokens would dominate the budget. - -## AI Pipeline: Composable Skills Architecture - -### LangGraph as Orchestrator + Skills as Modules - -Based on research documented in [LLM.md](LLM.md), the architecture should use **LangGraph as the orchestrator** and **Claude-style Skills as the domain modules**. This is the "Orchestrator vs. Module" pattern: - -- **LangGraph** manages the workflow: deterministic routing, state persistence, conditional edges, human-in-the-loop checkpoints, and recovery from failures at any step. -- **Skills** provide the domain expertise: each is a self-contained module with its own prompt template, input/output schema, error handling, and documentation. LangGraph nodes delegate to Skills for actual LLM work. - -This combination is stronger than either approach alone: -- Skills without LangGraph rely on non-deterministic LLM reasoning for orchestration — the model might skip steps or hallucinate new ones. -- LangGraph without Skills forces all prompt logic into graph nodes, creating a monolith that's hard to test or reuse. -- Together, LangGraph provides the deterministic "director" while Skills provide the swappable "actors." - -**Key LangGraph benefits for this project:** -- **State persistence:** If the ingestion pipeline fails at deduplication (step 3 of 5), LangGraph can resume from that checkpoint rather than reprocessing from scratch. -- **Conditional routing:** Skip summarization if relevance score is below threshold. Route single-article newsletters to entity extraction instead of link extraction. -- **Human-in-the-loop:** Pause for human review when confidence is low (e.g., ambiguous entity matches). -- **Multi-model routing:** Use cheaper models for high-volume steps and expensive models only where quality matters (see Multi-Model Strategy below). - -### Multi-Model Strategy - -The Skills architecture should be **model-agnostic**, not locked to Claude. Claude-style Skills (the `.claude/skills/` or `.github/skills/` folder structure with YAML frontmatter, `SKILL.md`, and supporting files) have been adopted as an open standard — GPT models can use them directly in VS Code, and self-hosted models may be able to as well. - -Design each Skill with a model-agnostic interface: -- The Skill defines the **prompt template**, **input/output schema**, and **evaluation criteria** -- The **model** is a configuration parameter, not hardcoded -- LangGraph's multi-model routing allows different nodes to use different models - -**Recommended model allocation** (see [VENDOR.md](VENDOR.md) for detailed rationale and API pricing): - -| Skill | Volume | Complexity | Dev Model (via OpenRouter) | Production Model (Ollama) | Why This Model | -|-------|--------|------------|---------------------------|--------------------------|----------------| -| Content Classification | High | Low | Llama 3.1 70B ($0.40/$0.40/M) | Llama 3.1 70B | Best instruction following for nuanced categories | -| Relevance Scoring | High | Medium | Embeddings primary; Qwen 2.5 72B for borderline ($0.12/$0.39/M) | Embeddings primary; Command R+ 104B | Dev: Qwen adequate for grounding. Prod: Command R+ purpose-built for RAG citation | -| Deduplication | High | Low | Embeddings primary; Qwen 2.5 72B for borderline | Embeddings primary; Command R+ 104B | Dev: Qwen adequate for comparison. Prod: Command R+ cross-referencing training | -| Summarization | Medium | High | Gemma 3 27B ($0.08/$0.16/M) | Gemma 3 27B | Clean, non-"AI-sounding" prose; fast at 27B | -| Theme Detection | Low (daily) | High | DeepSeek V3.2 ($0.26/$0.42/M) | DeepSeek V3 | Strong cross-document reasoning for pattern detection | -| Newsletter Extraction | Low | High | Qwen 2.5 72B ($0.12/$0.39/M) | Qwen 2.5 72B | Best structured output / JSON reliability | -| Entity Extraction | Low | Medium | Qwen 2.5 72B ($0.12/$0.39/M) | Qwen 2.5 72B | Accurate entity identification, low hallucination | - -**Key cost control principles:** -- Embeddings handle the primary workload for relevance scoring and deduplication. LLM calls only happen for borderline cases and explanations — during development these use Qwen 2.5 72B; in production, Command R+ 104B (self-hosted) provides superior RAG grounding at zero marginal cost. -- Summarization uses Gemma 3 27B instead of Claude Sonnet — 37x cheaper on input tokens. Quality validation via eval harness is a priority since summaries are the most user-facing output. -- All selected models are self-hostable via Ollama, providing a path to zero marginal LLM cost in production. Command R+ at 104B is the heaviest; the others fit comfortably on consumer GPU hardware. -- OpenRouter serves as the unified API gateway during development (single key, single billing), with a config-only switch to Ollama endpoints for production. -- During development, Qwen 2.5 72B substitutes for Command R+ in relevance scoring and deduplication to keep API costs minimal (~$2.30/month vs ~$6/month). The eval harness will validate whether Command R+'s specialized RAG grounding produces meaningfully better results before committing to its higher self-hosting requirements. - -### Embeddings: The Cost-Efficient Foundation - -**Embeddings** are dense vector representations of text — a way to convert an article, sentence, or paragraph into a list of numbers (typically 768-1536 dimensions) that captures its semantic meaning. Two texts about similar topics will have similar embeddings (measured by cosine similarity), even if they use different words. - -Embeddings are: -- **Cheap:** Orders of magnitude cheaper than LLM generation calls. Local embedding models (e.g., `sentence-transformers` or Ollama with `nomic-embed-text`) are free. -- **Fast:** A single embedding call takes milliseconds vs. seconds for an LLM generation. -- **Deterministic:** Same input always produces the same output. - -**How embeddings reduce LLM costs in this project:** -- **Relevance scoring:** Instead of asking Claude "is this article relevant to DevOps?" for every item, compute the article's embedding and measure cosine similarity against the newsletter's reference corpus embeddings in Qdrant. Only send borderline cases to an LLM for nuanced judgment. -- **Deduplication:** Compare new content embeddings against recent embeddings. High similarity (>0.92) = likely duplicate. No LLM call needed for the common case. -- **Theme detection:** Cluster recent content embeddings to identify topic groups. The LLM only needs to *label* the clusters, not discover them. - -**Embedding model options:** -- **Local (free):** `sentence-transformers/all-MiniLM-L6-v2`, Ollama with `nomic-embed-text` -- **API (cheap):** OpenAI `text-embedding-3-small` (~$0.02/1M tokens), Voyage AI, Cohere Embed - -### Claude-Style Skills as Portable Modules - -Each Skill follows the progressive disclosure pattern from [LLM.md](LLM.md): - -1. **Level 1 (Metadata):** YAML frontmatter with name, description, and trigger keywords. This is all the orchestrator sees during routing. -2. **Level 2 (Instructions):** The `SKILL.md` body with the standard operating procedure — step-by-step instructions, constraints, and references to supporting files. -3. **Level 3 (Resources):** `scripts/`, `references/`, and `assets/` subdirectories containing deterministic code, reference data, and templates. - -This structure ensures token efficiency (only load what's needed) and portability (Skills work across Claude, GPT, and potentially local models). - -### Skill Catalog - -Each skill is a standalone module with a defined interface. The model used is configurable per-skill (not hardcoded to Claude): - -1. **Content Classification:** Given raw content, classify as: technical article, tutorial, opinion piece, product announcement, event, release notes, other. Returns structured classification with confidence. -2. **Relevance Scoring:** Given content + a newsletter's topic description + reference corpus from the vector DB, score relevance 0-1 with explanation. Uses semantic similarity against reference embeddings as a signal alongside Claude's judgment. -3. **Deduplication / Clustering:** Given a new content item + recent embeddings from Qdrant, determine if this covers a topic already ingested. If so, pick the best version (highest authority source, most comprehensive coverage). -4. **Summarization:** Given an article, generate a newsletter-ready summary (2-3 sentences) that a non-technical editor could use directly or edit. -5. **Theme Detection:** Given all content ingested in the current period, identify 3-5 emerging themes and suggest them as newsletter section topics. -6. **Newsletter Email Extraction:** Given raw newsletter HTML, extract structured data: list of article links, titles, authors, brief descriptions, and the newsletter's name/date. -7. **Entity Extraction:** Given an article or newsletter, identify mentioned people, companies, and organizations. Suggest matches against existing entity profiles. - -### Default Pipeline Orchestration - -The standard content ingestion flow chains skills in order: - -``` -New Content → Classification → Relevance Scoring → (if relevant) → Deduplication → Summarization - (if not relevant) → archive, skip remaining steps -``` - -Theme Detection runs separately on a schedule (e.g., when the user opens the dashboard, or daily). - -Newsletter Email Extraction and Entity Extraction run when new newsletter emails arrive via Resend. The Newsletter Email Extraction skill should distinguish between curated/roundup newsletters (extract links and authors) and single-article newsletters (treat the author as the primary signal for entity authority scoring). This classification should happen at ingestion time, since even newsletters that are typically one format may have special editions in the other format. - -> **On LangGraph:** The orchestrator is implemented as a LangGraph StateGraph. This gives conditional routing (skip summarization if relevance is low, route single-article newsletters differently), state persistence (resume from failures), human-in-the-loop checkpoints, and multi-model routing. Each graph node delegates to a standalone Skill module, keeping the Skills framework-agnostic and independently testable. The LangGraph layer is an implementation detail of the orchestrator — the Skills themselves have no dependency on it. - -### Vector DB Role - -Qdrant stores embeddings of all ingested content. Used by multiple skills: - -- **Deduplication skill:** similarity search to find near-duplicate content -- **Relevance scoring skill:** compare against reference corpus embeddings for the newsletter's topic -- **Theme detection skill:** cluster recent embeddings to identify emerging topic groups -- **Entity extraction skill:** compare bio text / writing samples for entity matching suggestions - -## Scope and Phasing - -**The single biggest risk with this project is scope.** As described, it encompasses: 6+ platform integrations, entity resolution, authority scoring, an LLM analysis pipeline, a vector database, a relational database, a React UI, a plugin system, and newsletter generation assistance. That's easily 3-6 months of focused work for one person. - -A phased approach is essential: - -### Phase 1: MVP — Content Ingestion + Basic Surfacing - -**Goal:** Ingest content from easy sources, store it, and surface the most relevant items for a newsletter edition. - -- RSS feed plugin (track 20-30 blogs/sites manually configured) -- Reddit plugin (monitor 3-5 subreddits) -- Basic entity model in Postgres (manually created profiles) -- Vector DB (Qdrant) for content embeddings -- Composable skills: classification, relevance scoring, summarization (each standalone, chained by default orchestrator) -- Minimal UI: dashboard showing top content for the current period, sorted by relevance. Upvote/downvote buttons. -- Seed script that pre-populates with sample data for demo purposes (e.g., a month of RSS content from platform engineering blogs, a dozen parsed newsletters, pre-created entity profiles with linked accounts). This makes the project immediately demo-able without weeks of data accumulation. -- No authority scoring, no social media, no newsletter ingestion yet - -### Phase 2: Newsletter Ingestion + Authority Signals - -- Newsletter email ingestion via Resend with Claude-based extraction -- Subscription confirmation flow in UI -- Authority scoring based on newsletter mention frequency -- Bluesky plugin (easiest social media API) -- Deduplication skill added to pipeline -- Entity extraction skill: auto-suggest new entities from ingested newsletters -- Entity profile linking (still mostly manual, with LLM-suggested matches) - -### Phase 3: Expanded Sources + Intelligence - -- Mastodon plugin -- Trend velocity detection -- Theme suggestion for upcoming newsletter -- Source diversity analysis -- Original content idea generation ("Our Insight" suggestions) - -### Phase 4: Polish + Advanced Features - -- LinkedIn integration (if feasible) -- Automated entity discovery from ingested content -- Full authority scoring model with multiple signals -- Newsletter draft generation / templating (connect to GENRES.md layout types) - -## Feedback Loop Design - -When a user upvotes / downvotes surfaced content, the feedback needs to flow back into the system to improve future relevance. Four options were considered: - -### Option A: Adjust Authority Scores of the Source Entity - -The upvote/downvote modifies the authority score of the entity (person/company) that produced the content. - -| | | -|---|---| -| **Benefits** | Simple to implement. A single numeric adjustment per entity per tenant. Naturally surfaces more content from authors the editor trusts. Compounds over time — consistently good sources float up. | -| **Drawbacks** | Coarse-grained: penalizes/rewards the *source* when the issue might be the *topic*. An author you generally like might write one irrelevant article — downvoting shouldn't tank their authority. Doesn't capture topical preferences at all. | - -### Option B: Store Feedback as Labeled Training Data - -Store each upvote/downvote as a labeled example (content features → relevant/not-relevant). Periodically use these examples to refine the relevance scoring prompt via few-shot examples or fine-tuning. - -| | | -|---|---| -| **Benefits** | Captures nuanced preferences that authority scores miss (topic, format, depth). Gets smarter over time with more data. The labeled dataset is reusable for evaluation and regression testing. | -| **Drawbacks** | Requires a meaningful volume of feedback before it improves anything (~50-100+ labeled examples). Prompt engineering with few-shot examples has context window limits. Fine-tuning is expensive and operationally complex for a multi-tenant system. Latency between feedback and improvement (batch retraining). | - -### Option C: Adjust Vector Similarity Weights - -Use feedback to shift the tenant's "topic centroid" in embedding space. Upvoted content's embeddings pull the centroid toward similar content; downvoted content pushes it away. - -| | | -|---|---| -| **Benefits** | Mathematically elegant. Works at the semantic level — captures what topics and styles the editor prefers, not just which sources. Fast to compute. No LLM calls needed for the feedback loop itself. | -| **Drawbacks** | Hardest to implement correctly. Requires careful tuning of learning rate (how much does one vote shift the centroid?). Can drift in unintuitive ways with sparse feedback. Difficult to explain to users ("why did this get ranked higher?"). Debugging is opaque. | - -### Implementation: Combination - -Use a weighted combination, phased over time: - -- **Phase 1 (MVP):** Option A only — adjust entity authority scores. Simple, immediate, and covers the most common case ("I trust this author" / "this source is noise"). -- **Phase 2:** Add Option C — build a per-tenant topic centroid from upvoted content embeddings. Use cosine similarity to the centroid as an additional relevance signal alongside authority score. -- **Phase 3:** Add Option B — accumulate labeled examples and use them as few-shot examples in the relevance scoring prompt. This adds the nuanced "what does this specific editor value?" signal. - -The final relevance score becomes: `weighted_sum(embedding_similarity, authority_score, feedback_adjusted_score)` with weights tunable per tenant. - -## Composable UI Design - -Here are possibilities for non-technical users chaining primitives into workflows from simplest to most ambitious: - -### Level 1: Contextual Skill Actions (MVP) - -Individual skills are exposed as actions on content items throughout the UI. When viewing any article, the user sees action buttons or a right-click menu: - -- **"Summarize"** — runs the Summarization skill on this article, shows the result inline -- **"Find related"** — runs the Deduplication / similarity skill to surface related content -- **"Extract entities"** — runs Entity Extraction and shows suggested people / companies to add -- **"Check relevance"** — runs Relevance Scoring and explains why this item was ranked where it is - -This is the simplest approach and covers most real use cases. It demonstrates that skills are independently invocable, not locked inside the pipeline. - -### Level 2: Multi-Step Skill Chaining - -A "Run Skills" panel where the user selects a content item, then picks an ordered sequence of skills to run. The output of each step feeds into the next. Example workflow a user might build: - -1. Select an article → **Summarize** → **Entity Extraction** → auto-create entity profiles from the summary -2. Select a newsletter email → **Newsletter Extraction** → **Relevance Scoring** on each extracted link → show only relevant links - -The UI would show a simple linear pipeline builder (drag skills into a sequence) with a preview of each step's output. - -### Level 3: Saved Workflows / Templates - -Let users save their custom skill chains as named workflows they can re-run. Example: "My weekly prep" = run Theme Detection → show top 20 items → auto-summarize the top 5. These saved workflows would also serve as great demo material. - -**Recommendation:** Start with Level 1 for the MVP — it's low implementation cost and directly demonstrates composability. Level 2 is a strong Phase 3/4 feature. Level 3 is a nice-to-have stretch goal. - -## Monitoring and Observability - -### Recommended Approach: Django-Native with a Lightweight External Stack - -For a portfolio project at this scale, a hybrid approach balances production-readiness with operational simplicity: - -**Built into Django + the React UI:** -- **Ingestion health dashboard:** A dedicated UI page showing the status of each data source plugin — last successful fetch, error count, current status (healthy/degraded/failing). Backed by a Django model that logs each ingestion run. -- **Human review queue:** A UI page listing items flagged by skills as low-confidence (e.g., ambiguous entity matches, classification confidence < 0.6, articles that might be duplicates but aren't certain). Users can resolve these manually. -- **Skill execution logs:** A Django model logging each skill invocation — input, output, model used, latency, success/failure, confidence score. Viewable in the UI as a filterable activity log. - -**External lightweight stack (optional, Phase 3+):** -- **Structured logging:** Use Python's `structlog` to emit JSON logs from the Django app and Celery workers. These can be consumed by any log aggregator. -- **Prometheus metrics:** Expose key metrics via `django-prometheus` — skill execution latency, ingestion success/failure rates, queue depth, API response times. Minimal setup if deploying to Kubernetes. -- **Grafana dashboards:** Connect to Prometheus for operational dashboards. Useful but not essential for MVP. - -**Skip for this project:** Full ELK/Loki stack, distributed tracing (OpenTelemetry), APM tools. These are overkill for a single-server portfolio project. - -The Django-native monitoring is the priority — it's part of the product UX and demonstrates the "production agents that know when to escalate" principle. External observability is infrastructure polish for later. - -## Deployment - Docker Compose for MVP, Kubernetes-Ready - -**Phase 1 — Docker Compose on a single VM:** - -A `docker-compose.yml` that runs all services: -- Django app (gunicorn) -- Celery worker + Celery beat (scheduled tasks) -- PostgreSQL -- Redis (Celery broker + cache) -- Qdrant (vector DB) -- Nginx (reverse proxy + static files) - -Deploy to a single VPS (e.g., Hetzner, DigitalOcean, Railway). Total cost: ~$20-40/month. - -**Phase 2+ — Kubernetes-ready:** - -The Docker Compose setup naturally translates to Kubernetes manifests or a Helm chart: -- Each service becomes a Deployment + Service -- PostgreSQL and Redis can move to managed services (RDS, ElastiCache) or remain in-cluster -- Qdrant can move to Qdrant Cloud or remain self-hosted -- Celery workers scale independently via HPA -- Ingress replaces Nginx - -**What to build now (even for Docker Compose):** -- Dockerfiles for the Django app and Celery worker -- Environment-based configuration (12-factor app style) — database URLs, API keys, model endpoints all via environment variables -- Health check endpoints (`/healthz`, `/readyz`) for each service -- A `just` file with common commands (`make dev`, `make seed`, `make test`) - -**Skip for now:** Terraform/IaC for infrastructure provisioning, CI/CD pipelines (GitHub Actions is fine for builds), multi-region deployment. - -## Open Questions - -1. **Multi-model compatibility testing.** ~~The Skills architecture is designed to be model-agnostic, and GPT models are confirmed to work with Claude-style Skills in VS Code. We should evaluate which local/self-hosted models (via Ollama) can effectively consume the progressive disclosure Skill format. This affects our cost model — if local models handle Skills well, the high-volume commodity tasks (classification, relevance scoring) can run at zero marginal cost. **Action:** Set up a test harness that runs each Skill against multiple models and compares output quality.~~ - - **Resolved.** Research on self-hosted model compatibility is documented in the "Claude Skills with Self-Hosted LLMs" section of [LLM.md](LLM.md). Key findings: the progressive disclosure model is a system design pattern, not a model feature — it requires a "Skill Loader" runtime layer in LangGraph. Best local model candidates are **Qwen 2.5 72B** (strongest tool discovery), **Command R+** (best SOP adherence), and **Llama 3.1 70B** (reliable generalist). The LangGraph implementation handles all three tiers: a registry init node for Level 1 metadata, a `load_skill()` tool call for Level 2, and a `run_script()` tool call for Level 3. Remaining action: build the test harness to benchmark each Skill against these models and compare output quality — this is an implementation task, not an open design question. - -2. **Feedback loop implementation details.** The phased approach (entity authority → topic centroid → few-shot examples) is the plan, but concrete implementation questions remain: - - What's the right learning rate for authority score adjustments? (e.g., +0.1 per upvote, -0.05 per downvote, with decay?) - - How many feedback signals before the topic centroid becomes useful? - - Should downvotes have stronger weight than upvotes (negativity bias)? - - **Decision: Expose as admin-configurable controls.** Rather than hardcoding these values, the admin UI should expose them as adjustable settings per tenant. This lets each newsletter editor (or the system administrator) tune the feedback loop to their domain without code changes. - - **Admin controls to expose:** - - | Control | Default | Range | Notes | - |---------|---------|-------|-------| - | Upvote authority weight | +0.1 | 0.01–0.5 | How much a single upvote boosts an entity's authority score | - | Downvote authority weight | -0.05 | -0.5–0 | How much a single downvote penalizes. Default asymmetric (weaker than upvote) to avoid a single bad article tanking a good source | - | Authority score decay rate | 0.95/month | 0.8–1.0 | Multiplicative monthly decay so stale entities don't dominate. 1.0 = no decay | - | Centroid learning rate | 0.05 | 0.01–0.2 | How aggressively the topic centroid shifts per feedback signal | - | Centroid activation threshold | 30 signals | 10–100 | Minimum feedback count before the centroid influences relevance scoring | - | Negativity bias multiplier | 1.5x | 1.0–3.0 | Whether downvotes shift the centroid more than upvotes (1.0 = symmetric) | - - **Implementation approach:** Store these as a JSON config field on the Tenant model with sensible defaults. The admin UI renders them as sliders/number inputs with min/max bounds. Include a "Reset to defaults" button. Add a "Preview impact" feature in Phase 3 that shows how changing a value would re-rank the current dashboard. - - **Why asymmetric defaults (upvote > downvote)?** A new system has sparse data. If downvotes are too aggressive, a single misfire (e.g., a great author writes one off-topic piece) permanently buries them. It's safer to let good signals accumulate gradually and let bad signals have a gentler effect. Users who want stronger negativity bias can increase the multiplier. - -3. **Composable UI scope for MVP.** Level 1 (contextual skill actions) is planned for MVP. - - ### Skill Actions: UI Implementation Sketch - - At its core, this is a request-response pattern with a status lifecycle. Each skill action follows the same flow: user triggers → loading state → result displayed → user acts on result. - - **Content Detail View (the primary interaction surface):** - - When a user views a content item (article, post, etc.) from the dashboard, the detail panel includes a "Skills" toolbar: - - ``` - ┌─────────────────────────────────────────────────────┐ - │ Article: "Kubernetes 1.31 Release Highlights" │ - │ Source: kubernetes.io/blog • 2 days ago │ - │ Relevance: 0.87 • Authority: 0.72 │ - ├─────────────────────────────────────────────────────┤ - │ [📝 Summarize] [🔍 Find Related] [👤 Extract Entities] [📊 Explain Relevance] │ - ├─────────────────────────────────────────────────────┤ - │ ▼ Summary (generated 2 min ago) │ - │ ┌─────────────────────────────────────────────────┐ │ - │ │ Kubernetes 1.31 introduces native sidecar │ │ - │ │ container support and graduates several... │ │ - │ │ [Copy] [Regenerate] │ │ - │ └─────────────────────────────────────────────────┘ │ - │ │ - │ ▼ Extracted Entities (generated 5 min ago) │ - │ ┌─────────────────────────────────────────────────┐ │ - │ │ • Tim Hockin (Google) — matched existing entity │ │ - │ │ • SIG Node — new, [Add to Entities] │ │ - │ └─────────────────────────────────────────────────┘ │ - └─────────────────────────────────────────────────────┘ - ``` - - **Backend implementation:** - - - **API endpoint:** `POST /api/v1/tenants/{tenant_id}/contents/{id}/skills/{skill_name}/` — triggers the skill, returns a `SkillResult` object with status (pending/running/completed/failed), result data, and metadata (model used, latency, confidence). - - **Async execution:** Skills that take >1s (summarization, entity extraction) should run via Celery and return a `202 Accepted` with a result ID. The frontend polls or uses WebSocket for completion. Classification and relevance explanations are fast enough for synchronous response. - - **Error handling:** If a skill fails (model timeout, malformed input), the API returns the error in the `SkillResult` with status `failed` and a user-readable message. The UI shows an inline error with a "Retry" button. No silent failures. - - **Django models:** - - ```python - class SkillResult(models.Model): - content = models.ForeignKey(Content, on_delete=models.CASCADE, related_name='skill_results') - tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE) - skill_name = models.CharField(max_length=64) # e.g. "summarization", "entity_extraction" - status = models.CharField(choices=STATUS_CHOICES) # pending, running, completed, failed - result_data = models.JSONField(null=True) # skill-specific structured output - error_message = models.TextField(blank=True) - model_used = models.CharField(max_length=64) # e.g. "claude-sonnet-4", "ollama/mistral" - latency_ms = models.IntegerField(null=True) - confidence = models.FloatField(null=True) - created_at = models.DateTimeField(auto_now_add=True) - - class Meta: - indexes = [ - models.Index(fields=['content', 'skill_name']), - models.Index(fields=['tenant', 'created_at']), - ] - ``` - - **Frontend implementation:** - - - React component: `` renders the skill buttons for a content item. Each button triggers the API call and manages its own loading/result/error state. - - Results render inline below the toolbar in collapsible sections. Multiple skill results can be open simultaneously. - - Loading state: button shows a spinner, disabled to prevent duplicate submissions. - - Each result section has action buttons relevant to that skill (Copy, Regenerate, Add to Entities, etc.). - - ### Persistence: Should Skill Results Be Saved? - - **Recommendation: Yes, persist by default.** The cost is low and the benefits are significant. - - **What we gain by persisting:** - - - **No redundant LLM calls.** If a user summarizes an article on Monday and returns Wednesday, the summary is already there. Without persistence, they'd re-trigger the skill and pay for another LLM call. At $0.003-0.01 per summarization, this adds up across hundreds of articles. - - **Audit trail.** The skill execution log (model used, latency, confidence) is valuable for debugging relevance issues and tuning the pipeline. "Why was this article ranked so high?" becomes answerable by inspecting the persisted relevance explanation. - - **Cross-user visibility (future).** If multiple editors share a tenant, one editor's entity extraction results are visible to others without re-running. - - **Evaluation dataset.** Persisted results + user feedback (did they use the summary? edit it? discard it?) become training data for improving skill prompts over time. - - **Offline/async use.** Background jobs can pre-run skills (e.g., auto-summarize all items above relevance 0.8) and have results ready when the user opens the dashboard. - - **What we lose by persisting:** - - - **Staleness risk.** A summary generated with an older model version or before the article was updated might be misleading. Mitigation: show the generation timestamp, and offer a "Regenerate" button that creates a new result (keeping the old one for comparison). - - **Storage cost.** See estimates below — this is negligible. - - **Complexity.** Need to handle versioning (multiple results for the same content+skill pair) and cleanup. But the `SkillResult` model above already handles this naturally. - - **Storage estimates:** - - | Skill | Avg result size | Volume (1 tenant, 1 year) | Storage | - |-------|----------------|---------------------------|---------| - | Summarization | ~500 bytes (2-3 sentences) | 2,000 articles × 1 result | ~1 MB | - | Entity Extraction | ~1 KB (list of entities + metadata) | 2,000 articles × 1 result | ~2 MB | - | Relevance Explanation | ~300 bytes | 5,000 articles × 1 result | ~1.5 MB | - | Classification | ~100 bytes | 5,000 articles × 1 result | ~0.5 MB | - | Find Related | ~2 KB (list of related item IDs + scores) | 500 manual invocations | ~1 MB | - | **Total per tenant/year** | | | **~6 MB** | - - At ~6 MB per tenant per year in the `result_data` JSON field, storage is a non-issue even at hundreds of tenants. PostgreSQL handles this comfortably. The content embeddings in Qdrant (768-1536 floats per article) dwarf this by orders of magnitude. - - **Versioning policy:** Keep the latest result per content+skill pair as the "active" result. If the user clicks "Regenerate," create a new `SkillResult` row and mark the previous one as superseded (a `superseded_by` FK or a boolean). Purge superseded results older than 90 days via the same Celery cleanup task that handles content retention. - -## Reference Links - -- See [NOTES.md](NOTES.md) for research on existing content discovery / curation tools -- See [GENRES.md](GENRES.md) for newsletter format types and layout templates -- See [IMPLEMENTATION.md](IMPLEMENTATION.md) for additional implementation notes -- See [LLM.md](LLM.md) for research on Claude Skills architecture, LangGraph integration, and multi-model considerations -- See [VENDOR.md](VENDOR.md) for per-skill model selection, rationale, and API pricing via OpenRouter diff --git a/justfile b/justfile index 7b0c7717..8ba7da9c 100644 --- a/justfile +++ b/justfile @@ -59,6 +59,14 @@ lint: pre-commit run --all-files trailing-whitespace python3 manage.py check +lint-fix: + if [ ! -f .env ]; then cp .env.example .env; fi + ruff check manage.py core newsletter_maker tests --fix + djlint core/templates --reformat + pre-commit run --all-files end-of-file-fixer + pre-commit run --all-files trailing-whitespace + just lint + test: python3 -m pytest diff --git a/newsletter_maker/settings/base.py b/newsletter_maker/settings/base.py index a61ef474..37280aaa 100644 --- a/newsletter_maker/settings/base.py +++ b/newsletter_maker/settings/base.py @@ -1,9 +1,10 @@ import os import sys from pathlib import Path -from django.utils.translation import gettext_lazy as _ -from django.templatetags.static import static + import dj_database_url +from django.templatetags.static import static +from django.utils.translation import gettext_lazy as _ from dotenv import load_dotenv BASE_DIR = Path(__file__).resolve().parent.parent.parent @@ -57,6 +58,8 @@ def env_list(name: str, default: str = "") -> list[str]: "django.contrib.staticfiles", "import_export", "rest_framework", + "drf_spectacular", + "drf_standardized_errors", ] MIDDLEWARE = [ @@ -123,6 +126,12 @@ def env_list(name: str, default: str = "") -> list[str]: "rest_framework.authentication.SessionAuthentication", "rest_framework.authentication.BasicAuthentication", ], + "DEFAULT_SCHEMA_CLASS": "drf_standardized_errors.openapi.AutoSchema", + "EXCEPTION_HANDLER": "drf_standardized_errors.handler.exception_handler", +} + +DRF_STANDARDIZED_ERRORS = { + "ALLOWED_ERROR_STATUS_CODES": ["400", "403", "404"], } SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") @@ -146,4 +155,51 @@ def env_list(name: str, default: str = "") -> list[str]: ], "SITE_ICON": lambda request: static("core/logo.svg"), "SITE_SYMBOL": "newsletter", -} \ No newline at end of file +} + +# Add metadata for Swagger UI +SPECTACULAR_SETTINGS = { + "TITLE": "Newsletter Maker API", + "DESCRIPTION": "API documentation for the newsletter maker app", + "VERSION": "1.0.0", + "SERVE_INCLUDE_SCHEMA": False, + "POSTPROCESSING_HOOKS": ["drf_standardized_errors.openapi_hooks.postprocess_schema_enums"], + "ENUM_NAME_OVERRIDES": { + "ValidationErrorEnum": "drf_standardized_errors.openapi_serializers.ValidationErrorEnum.choices", + "ClientErrorEnum": "drf_standardized_errors.openapi_serializers.ClientErrorEnum.choices", + "ServerErrorEnum": "drf_standardized_errors.openapi_serializers.ServerErrorEnum.choices", + "ParseErrorCodeEnum": "drf_standardized_errors.openapi_serializers.ParseErrorCodeEnum.choices", + "ErrorCode403Enum": "drf_standardized_errors.openapi_serializers.ErrorCode403Enum.choices", + "ErrorCode404Enum": "drf_standardized_errors.openapi_serializers.ErrorCode404Enum.choices", + }, + "TAGS": [ + { + "name": "Tenant Management", + "description": "Create tenants and manage tenant-specific configuration for newsletter workspaces.", + }, + { + "name": "Entity Catalog", + "description": "Manage tracked people, companies, and organizations associated with a tenant.", + }, + { + "name": "Content Library", + "description": "Browse and maintain ingested content items that feed newsletter generation and ranking.", + }, + { + "name": "AI Processing", + "description": "Inspect AI skill execution results, model outputs, and confidence metadata for tenant content.", + }, + { + "name": "Feedback", + "description": "Capture editorial feedback signals that influence ranking and future recommendation quality.", + }, + { + "name": "Ingestion", + "description": "Configure source plugins and review ingestion runs for each tenant.", + }, + { + "name": "Review Queue", + "description": "Review borderline or low-confidence content items that need human resolution.", + }, + ], +} diff --git a/newsletter_maker/urls.py b/newsletter_maker/urls.py index 159b566f..d7b63242 100644 --- a/newsletter_maker/urls.py +++ b/newsletter_maker/urls.py @@ -1,7 +1,10 @@ +from django.conf import settings from django.contrib import admin +from django.contrib.staticfiles.urls import staticfiles_urlpatterns from django.shortcuts import redirect from django.urls import include, path from django.views.generic.base import RedirectView +from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView def root_redirect_view(request): @@ -12,5 +15,10 @@ def root_redirect_view(request): path("admin/", admin.site.urls), path("favicon.ico", RedirectView.as_view(url="/static/core/favicon.ico", permanent=True)), path("api/v1/", include(("core.api_urls", "api"), namespace="v1")), + path("api/schema/", SpectacularAPIView.as_view(), name="schema"), + path("api/docs/", SpectacularSwaggerView.as_view(url_name="schema"), name="swagger-ui"), path("", include("core.urls")), ] + +if settings.DEBUG: + urlpatterns += staticfiles_urlpatterns() diff --git a/requirements.txt b/requirements.txt index eb89c90d..95bf1b90 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,11 +9,15 @@ django-unfold==0.90.0 Django==6.0.4 djangorestframework==3.17.1 djlint==1.36.4 +drf-nested-routers==0.95.0 +drf-spectacular==0.29.0 +drf-standardized-errors==0.15.0 editorconfig==0.17.1 feedparser==6.0.12 gunicorn==25.3.0 httpx==0.28.1 identify==2.6.19 +inflection==0.5.1 jsbeautifier==1.15.4 librt==0.9.0 mypy==1.20.2 @@ -32,3 +36,5 @@ ruff==0.15.12 sentence-transformers==5.4.1 structlog==25.5.0 types-pyyaml==6.0.12.20260408 +uritemplate==4.2.0 +watchdog==6.0.0