diff --git a/core/api.py b/core/api.py index 4f4ee912..52ee8b73 100644 --- a/core/api.py +++ b/core/api.py @@ -35,6 +35,8 @@ EntityCandidate, EntityMention, IngestionRun, + IntakeAllowlist, + NewsletterIntake, Project, ProjectConfig, ReviewQueue, @@ -42,8 +44,10 @@ SourceConfig, TopicCentroidSnapshot, UserFeedback, + generate_project_intake_token, ) from core.serializers import ( + BlueskyCredentialsSerializer, ContentSerializer, EntityAuthoritySnapshotSerializer, EntityCandidateMergeSerializer, @@ -51,6 +55,8 @@ EntityMentionSummarySerializer, EntitySerializer, IngestionRunSerializer, + IntakeAllowlistSerializer, + NewsletterIntakeSerializer, ProjectConfigSerializer, ProjectSerializer, ReviewQueueSerializer, @@ -104,11 +110,35 @@ "group": 3, "topic_description": "Coverage of developer tools, model releases, and applied AI workflows.", "content_retention_days": 180, + "intake_token": "project-token-123", + "intake_enabled": True, + "has_bluesky_credentials": True, + "bluesky_handle": "aiweekly.bsky.social", + "bluesky_is_active": True, + "bluesky_last_verified_at": "2026-04-26T13:00:00Z", + "bluesky_last_error": "", "created_at": "2026-04-26T12:00:00Z", }, response_only=True, ) +BLUESKY_CREDENTIALS_RESPONSE_EXAMPLE = OpenApiExample( + "Bluesky Credentials Response", + value={ + "id": 1, + "project": 1, + "handle": "aiweekly.bsky.social", + "pds_url": "", + "is_active": True, + "has_stored_credential": True, + "last_verified_at": "2026-04-26T13:00:00Z", + "last_error": "", + "created_at": "2026-04-26T12:30:00Z", + "updated_at": "2026-04-26T13:00:00Z", + }, + response_only=True, +) + SOURCE_CONFIG_CREATE_REQUEST_EXAMPLE = OpenApiExample( "Create RSS Source Request", value={ @@ -594,7 +624,7 @@ class ProjectViewSet(viewsets.ModelViewSet): """Manage projects accessible through the current user's group memberships.""" serializer_class = ProjectSerializer - queryset = Project.objects.select_related("group") + queryset = Project.objects.select_related("group", "bluesky_credentials") lookup_url_kwarg = "id" def get_queryset(self): @@ -602,6 +632,26 @@ def get_queryset(self): return self.queryset.filter(group__user=self.request.user).distinct() + @extend_schema( + summary="Rotate newsletter intake token", + description=( + "Generate a fresh project-specific newsletter intake token and return the " + "updated project payload." + ), + tags=["Project Management"], + request=None, + responses={200: ProjectSerializer, 403: AUTHENTICATION_REQUIRED_RESPONSE}, + ) + @action(detail=True, methods=["post"], url_path="rotate-intake-token") + def rotate_intake_token(self, request, *args, **kwargs): + """Generate a fresh intake token for the selected project.""" + + project = self.get_object() + project.intake_token = generate_project_intake_token() + project.save(update_fields=["intake_token"]) + serializer = self.get_serializer(project) + return Response(serializer.data) + @extend_schema( summary="Verify Bluesky credentials", description=( @@ -1013,6 +1063,75 @@ class IngestionRunViewSet(ProjectOwnedQuerysetMixin, viewsets.ModelViewSet): queryset = IngestionRun.objects.select_related("project") +@document_project_owned_viewset( + resource_plural="Bluesky credentials", + resource_singular="Bluesky credentials", + create_description=( + "Create Bluesky credentials for the selected project. The app password is " + "accepted write-only and is never returned in API responses." + ), + tag="Ingestion", + action_overrides=build_crud_action_overrides( + BlueskyCredentialsSerializer, + resource_plural="Bluesky credentials for the selected project", + resource_singular="Bluesky credentials", + retrieve_examples=[BLUESKY_CREDENTIALS_RESPONSE_EXAMPLE], + ), +) +class BlueskyCredentialsViewSet(ProjectOwnedQuerysetMixin, viewsets.ModelViewSet): + """Manage project-scoped Bluesky credentials.""" + + serializer_class = BlueskyCredentialsSerializer + queryset = BlueskyCredentials.objects.select_related("project") + + def get_queryset(self): + """Restrict credentials to the selected project and current user.""" + + return super().get_queryset().order_by("-updated_at") + + +@document_project_owned_viewset( + resource_plural="intake allowlist entries", + resource_singular="intake allowlist entry", + create_description=( + "Create a new confirmed or pending sender allowlist entry for the selected " + "project's newsletter intake workflow." + ), + tag="Ingestion", + action_overrides=build_crud_action_overrides( + IntakeAllowlistSerializer, + resource_plural="intake allowlist entries for the selected project", + resource_singular="intake allowlist entry", + ), +) +class IntakeAllowlistViewSet(ProjectOwnedQuerysetMixin, viewsets.ModelViewSet): + """Manage newsletter sender allowlist entries for a project.""" + + serializer_class = IntakeAllowlistSerializer + queryset = IntakeAllowlist.objects.select_related("project") + + +@document_project_owned_viewset( + resource_plural="newsletter intake entries", + resource_singular="newsletter intake entry", + create_description=( + "Newsletter intake entries are created by inbound email processing and are " + "exposed read-only for audit and troubleshooting." + ), + tag="Ingestion", + action_overrides=build_crud_action_overrides( + NewsletterIntakeSerializer, + resource_plural="newsletter intake entries for the selected project", + resource_singular="newsletter intake entry", + ), +) +class NewsletterIntakeViewSet(ProjectOwnedQuerysetMixin, viewsets.ReadOnlyModelViewSet): + """Inspect inbound newsletter history for a project.""" + + serializer_class = NewsletterIntakeSerializer + queryset = NewsletterIntake.objects.select_related("project") + + @document_project_owned_viewset( resource_plural="source configurations", resource_singular="source configuration", diff --git a/core/api_urls.py b/core/api_urls.py index 92db1b03..e1c70538 100644 --- a/core/api_urls.py +++ b/core/api_urls.py @@ -2,10 +2,13 @@ from rest_framework_nested.routers import NestedSimpleRouter from core.api import ( + BlueskyCredentialsViewSet, ContentViewSet, EntityCandidateViewSet, EntityViewSet, IngestionRunViewSet, + IntakeAllowlistViewSet, + NewsletterIntakeViewSet, ProjectConfigViewSet, ProjectViewSet, ReviewQueueViewSet, @@ -38,6 +41,21 @@ project_router.register( r"ingestion-runs", IngestionRunViewSet, basename="project-ingestion-run" ) +project_router.register( + r"bluesky-credentials", + BlueskyCredentialsViewSet, + basename="project-bluesky-credentials", +) +project_router.register( + r"intake-allowlist", + IntakeAllowlistViewSet, + basename="project-intake-allowlist", +) +project_router.register( + r"newsletter-intakes", + NewsletterIntakeViewSet, + basename="project-newsletter-intake", +) project_router.register( r"source-configs", SourceConfigViewSet, basename="project-source-config" ) diff --git a/core/serializers.py b/core/serializers.py index ef3fcafd..059cd3a4 100644 --- a/core/serializers.py +++ b/core/serializers.py @@ -9,6 +9,7 @@ from rest_framework import serializers from core.models import ( + BlueskyCredentials, Content, Entity, EntityAuthoritySnapshot, @@ -83,6 +84,12 @@ def __init__(self, *args, **kwargs): class ProjectSerializer(ProjectScopedSerializerMixin, serializers.ModelSerializer): """Serialize top-level project records.""" + has_bluesky_credentials = serializers.SerializerMethodField() + bluesky_handle = serializers.SerializerMethodField() + bluesky_is_active = serializers.SerializerMethodField() + bluesky_last_verified_at = serializers.SerializerMethodField() + bluesky_last_error = serializers.SerializerMethodField() + class Meta: model = Project fields = [ @@ -93,10 +100,52 @@ class Meta: "content_retention_days", "intake_token", "intake_enabled", + "has_bluesky_credentials", + "bluesky_handle", + "bluesky_is_active", + "bluesky_last_verified_at", + "bluesky_last_error", "created_at", ] read_only_fields = ["id", "created_at"] + def _get_bluesky_credentials(self, obj: Project): + """Return the project's stored Bluesky credentials, if configured.""" + + try: + return obj.bluesky_credentials + except Project.bluesky_credentials.RelatedObjectDoesNotExist: + return None + + def get_has_bluesky_credentials(self, obj: Project) -> bool: + """Return whether the project has stored Bluesky credentials.""" + + return self._get_bluesky_credentials(obj) is not None + + def get_bluesky_handle(self, obj: Project) -> str: + """Return the stored Bluesky handle, or an empty string.""" + + credentials = self._get_bluesky_credentials(obj) + return credentials.handle if credentials else "" + + def get_bluesky_is_active(self, obj: Project) -> bool: + """Return whether the stored Bluesky credentials are currently active.""" + + credentials = self._get_bluesky_credentials(obj) + return credentials.is_active if credentials else False + + def get_bluesky_last_verified_at(self, obj: Project): + """Return the last successful verification timestamp, if available.""" + + credentials = self._get_bluesky_credentials(obj) + return credentials.last_verified_at if credentials else None + + def get_bluesky_last_error(self, obj: Project) -> str: + """Return the latest Bluesky verification error, or an empty string.""" + + credentials = self._get_bluesky_credentials(obj) + return credentials.last_error if credentials else "" + class ProjectConfigSerializer( ProjectScopedSerializerMixin, serializers.ModelSerializer @@ -115,6 +164,81 @@ class Meta: read_only_fields = ["id", "project"] +class BlueskyCredentialsSerializer( + ProjectScopedSerializerMixin, serializers.ModelSerializer +): + """Serialize project-scoped Bluesky credentials without exposing secrets.""" + + app_password = serializers.CharField( + write_only=True, + required=False, + allow_blank=True, + trim_whitespace=False, + ) + has_stored_credential = serializers.SerializerMethodField() + + class Meta: + model = BlueskyCredentials + fields = [ + "id", + "project", + "handle", + "pds_url", + "is_active", + "has_stored_credential", + "app_password", + "last_verified_at", + "last_error", + "created_at", + "updated_at", + ] + read_only_fields = [ + "id", + "project", + "has_stored_credential", + "last_verified_at", + "last_error", + "created_at", + "updated_at", + ] + + def get_has_stored_credential(self, obj: BlueskyCredentials) -> bool: + """Return whether the project has an encrypted Bluesky credential stored.""" + + return obj.has_stored_credential() + + def validate(self, attrs): + """Require an app password when creating a credential record.""" + + attrs = super().validate(attrs) + app_password = attrs.get("app_password", "") + if self.instance is None and not app_password: + raise serializers.ValidationError( + {"app_password": "A Bluesky app credential is required."} + ) + return attrs + + def create(self, validated_data): + """Encrypt the submitted Bluesky app password before saving the record.""" + + app_password = validated_data.pop("app_password", "") + instance = super().create(validated_data) + if app_password: + instance.set_app_password(app_password) + instance.save(update_fields=["app_password_encrypted", "updated_at"]) + return instance + + def update(self, instance, validated_data): + """Keep the stored credential unless a replacement app password is submitted.""" + + app_password = validated_data.pop("app_password", "") + instance = super().update(instance, validated_data) + if app_password: + instance.set_app_password(app_password) + instance.save(update_fields=["app_password_encrypted", "updated_at"]) + return instance + + class EntitySerializer(ProjectScopedSerializerMixin, serializers.ModelSerializer): """Serialize tracked entities for a project.""" @@ -462,12 +586,15 @@ class IntakeAllowlistSerializer( ): """Serialize confirmed and pending newsletter sender allowlist entries.""" + is_confirmed = serializers.BooleanField(read_only=True) + class Meta: model = IntakeAllowlist fields = [ "id", "project", "sender_email", + "is_confirmed", "confirmed_at", "confirmation_token", "created_at", diff --git a/core/tests/test_api.py b/core/tests/test_api.py index 874a515a..65fca82f 100644 --- a/core/tests/test_api.py +++ b/core/tests/test_api.py @@ -17,6 +17,9 @@ EntityMention, FeedbackType, IngestionRun, + IntakeAllowlist, + NewsletterIntake, + NewsletterIntakeStatus, Project, ProjectConfig, ReviewQueue, @@ -99,6 +102,29 @@ def setUp(self): items_fetched=5, items_ingested=4, ) + self.owner_intake_allowlist = IntakeAllowlist.objects.create( + project=self.owner_project, + sender_email="sender@example.com", + ) + self.owner_newsletter_intake = NewsletterIntake.objects.create( + project=self.owner_project, + sender_email="sender@example.com", + subject="Owner Digest", + raw_text="See https://example.com/post", + message_id="owner-intake-1", + status=NewsletterIntakeStatus.EXTRACTED, + extraction_result={ + "method": "heuristic", + "items": [ + { + "url": "https://example.com/post", + "title": "Example Post", + "excerpt": "A short preview", + "position": 1, + } + ], + }, + ) self.owner_review_queue = ReviewQueue.objects.create( project=self.owner_project, content=self.owner_content, @@ -147,11 +173,46 @@ def test_project_list_requires_authentication(self): ) def test_project_list_is_scoped_to_request_user_groups(self): + BlueskyCredentials.objects.create( + project=self.owner_project, + handle="owner-project.bsky.social", + is_active=True, + last_error="", + ) + response = self.client.get(reverse("v1:project-list")) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(len(response.json()), 1) self.assertEqual(response.json()[0]["id"], self.owner_project.id) + self.assertEqual( + response.json()[0]["intake_token"], self.owner_project.intake_token + ) + self.assertFalse(response.json()[0]["intake_enabled"]) + self.assertTrue(response.json()[0]["has_bluesky_credentials"]) + self.assertEqual( + response.json()[0]["bluesky_handle"], "owner-project.bsky.social" + ) + self.assertTrue(response.json()[0]["bluesky_is_active"]) + self.assertEqual(response.json()[0]["bluesky_last_error"], "") + + def test_project_rotate_intake_token_returns_updated_project(self): + original_token = self.owner_project.intake_token + + response = self.client.post( + reverse( + "v1:project-rotate-intake-token", + kwargs={"id": self.owner_project.id}, + ), + format="json", + ) + + self.owner_project.refresh_from_db() + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertNotEqual(self.owner_project.intake_token, original_token) + self.assertEqual( + response.json()["intake_token"], self.owner_project.intake_token + ) def test_entity_list_is_scoped_to_request_user_project(self): response = self.client.get( @@ -235,6 +296,157 @@ def test_entity_mentions_action_returns_full_mention_history(self): self.assertEqual(response.json()[1]["id"], first_mention.id) self.assertEqual(response.json()[0]["content_title"], second_content.title) + def test_intake_allowlist_list_is_scoped_to_request_user_project(self): + other_allowlist = IntakeAllowlist.objects.create( + project=self.other_project, + sender_email="other@example.com", + ) + + response = self.client.get( + reverse( + "v1:project-intake-allowlist-list", + kwargs={"project_id": self.owner_project.id}, + ) + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.json()), 1) + self.assertEqual(response.json()[0]["id"], self.owner_intake_allowlist.id) + self.assertFalse(response.json()[0]["is_confirmed"]) + self.assertNotEqual(response.json()[0]["id"], other_allowlist.id) + + def test_intake_allowlist_create_and_delete_manage_project_senders(self): + create_response = self.client.post( + reverse( + "v1:project-intake-allowlist-list", + kwargs={"project_id": self.owner_project.id}, + ), + {"sender_email": "new-sender@example.com"}, + format="json", + ) + + self.assertEqual(create_response.status_code, status.HTTP_201_CREATED) + created_allowlist = IntakeAllowlist.objects.get( + project=self.owner_project, + sender_email="new-sender@example.com", + ) + self.assertEqual(create_response.json()["project"], self.owner_project.id) + self.assertFalse(create_response.json()["is_confirmed"]) + + delete_response = self.client.delete( + reverse( + "v1:project-intake-allowlist-detail", + kwargs={ + "project_id": self.owner_project.id, + "pk": created_allowlist.id, + }, + ) + ) + + self.assertEqual(delete_response.status_code, status.HTTP_204_NO_CONTENT) + self.assertFalse( + IntakeAllowlist.objects.filter(pk=created_allowlist.id).exists() + ) + + def test_bluesky_credentials_list_create_and_update_hide_stored_password(self): + list_response = self.client.get( + reverse( + "v1:project-bluesky-credentials-list", + kwargs={"project_id": self.owner_project.id}, + ) + ) + + self.assertEqual(list_response.status_code, status.HTTP_200_OK) + self.assertEqual(list_response.json(), []) + + create_response = self.client.post( + reverse( + "v1:project-bluesky-credentials-list", + kwargs={"project_id": self.owner_project.id}, + ), + { + "handle": "@Owner.Project.BSKY.social", + "pds_url": "https://pds.example.com/xrpc/", + "is_active": True, + "app_password": "app-password", + }, + format="json", + ) + + self.assertEqual(create_response.status_code, status.HTTP_201_CREATED) + credentials = BlueskyCredentials.objects.get(project=self.owner_project) + self.assertEqual(credentials.handle, "owner.project.bsky.social") + self.assertEqual(credentials.pds_url, "https://pds.example.com") + self.assertEqual(credentials.get_app_password(), "app-password") + self.assertTrue(create_response.json()["has_stored_credential"]) + self.assertNotIn("app_password", create_response.json()) + + update_response = self.client.patch( + reverse( + "v1:project-bluesky-credentials-detail", + kwargs={ + "project_id": self.owner_project.id, + "pk": credentials.id, + }, + ), + { + "handle": "updated.bsky.social", + "pds_url": "", + "is_active": False, + }, + format="json", + ) + + credentials.refresh_from_db() + self.assertEqual(update_response.status_code, status.HTTP_200_OK) + self.assertEqual(credentials.handle, "updated.bsky.social") + self.assertFalse(credentials.is_active) + self.assertEqual(credentials.get_app_password(), "app-password") + + def test_bluesky_credentials_create_requires_app_password(self): + response = self.client.post( + reverse( + "v1:project-bluesky-credentials-list", + kwargs={"project_id": self.owner_project.id}, + ), + { + "handle": "owner.bsky.social", + "pds_url": "", + "is_active": True, + "app_password": "", + }, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assert_standardized_validation_error(response.json(), "app_password") + + def test_newsletter_intake_list_returns_recent_project_history(self): + other_intake = NewsletterIntake.objects.create( + project=self.other_project, + sender_email="other@example.com", + subject="Other Digest", + raw_text="Another item", + message_id="other-intake-1", + ) + + response = self.client.get( + reverse( + "v1:project-newsletter-intake-list", + kwargs={"project_id": self.owner_project.id}, + ) + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.json()), 1) + self.assertEqual(response.json()[0]["id"], self.owner_newsletter_intake.id) + self.assertEqual(response.json()[0]["status"], NewsletterIntakeStatus.EXTRACTED) + self.assertEqual( + response.json()[0]["extraction_result"]["items"][0]["title"], + "Example Post", + ) + self.assertNotEqual(response.json()[0]["id"], other_intake.id) + def test_entity_list_supports_authority_score_ordering(self): second_entity = Entity.objects.create( project=self.owner_project, @@ -694,6 +906,18 @@ def test_authenticated_nested_list_endpoints_smoke(self): "v1:project-ingestion-run-list", kwargs={"project_id": self.owner_project.id}, ), + reverse( + "v1:project-bluesky-credentials-list", + kwargs={"project_id": self.owner_project.id}, + ), + reverse( + "v1:project-intake-allowlist-list", + kwargs={"project_id": self.owner_project.id}, + ), + reverse( + "v1:project-newsletter-intake-list", + kwargs={"project_id": self.owner_project.id}, + ), reverse( "v1:project-source-config-list", kwargs={"project_id": self.owner_project.id}, @@ -751,6 +975,20 @@ def test_authenticated_nested_detail_endpoints_smoke(self, queue_centroid_mock): "pk": self.owner_ingestion_run.id, }, ), + reverse( + "v1:project-intake-allowlist-detail", + kwargs={ + "project_id": self.owner_project.id, + "pk": self.owner_intake_allowlist.id, + }, + ), + reverse( + "v1:project-newsletter-intake-detail", + kwargs={ + "project_id": self.owner_project.id, + "pk": self.owner_newsletter_intake.id, + }, + ), reverse( "v1:project-source-config-detail", kwargs={ @@ -800,6 +1038,17 @@ def test_authenticated_nested_detail_endpoints_smoke(self, queue_centroid_mock): ) ) + credentials = BlueskyCredentials.objects.create( + project=self.owner_project, + handle="owner-project.bsky.social", + ) + detail_endpoints.append( + reverse( + "v1:project-bluesky-credentials-detail", + kwargs={"project_id": self.owner_project.id, "pk": credentials.id}, + ) + ) + for endpoint in detail_endpoints: with self.subTest(endpoint=endpoint): response = self.client.get(endpoint) diff --git a/frontend/src/app/admin/health/__tests__/page.test.tsx b/frontend/src/app/admin/health/__tests__/page.test.tsx index da19bbd0..7c030e23 100644 --- a/frontend/src/app/admin/health/__tests__/page.test.tsx +++ b/frontend/src/app/admin/health/__tests__/page.test.tsx @@ -7,18 +7,21 @@ import type { Project, SourceConfig, TopicCentroidObservabilitySummary, + TopicCentroidSnapshot, } from "@/lib/types" const { getProjectIngestionRunsMock, getProjectsMock, getProjectSourceConfigsMock, + getProjectTopicCentroidSnapshotsMock, getProjectTopicCentroidSummaryMock, selectProjectMock, } = vi.hoisted(() => ({ getProjectIngestionRunsMock: vi.fn(), getProjectsMock: vi.fn(), getProjectSourceConfigsMock: vi.fn(), + getProjectTopicCentroidSnapshotsMock: vi.fn(), getProjectTopicCentroidSummaryMock: vi.fn(), selectProjectMock: vi.fn(), })) @@ -59,6 +62,7 @@ vi.mock("@/lib/api", () => ({ getProjectIngestionRuns: getProjectIngestionRunsMock, getProjects: getProjectsMock, getProjectSourceConfigs: getProjectSourceConfigsMock, + getProjectTopicCentroidSnapshots: getProjectTopicCentroidSnapshotsMock, getProjectTopicCentroidSummary: getProjectTopicCentroidSummaryMock, })) @@ -130,6 +134,23 @@ function createTopicCentroidSummary( } } +function createTopicCentroidSnapshot( + overrides: Partial = {}, +): TopicCentroidSnapshot { + return { + id: 5, + project: 1, + computed_at: "2026-04-28T08:00:00Z", + centroid_active: true, + feedback_count: 12, + upvote_count: 10, + downvote_count: 2, + drift_from_previous: 0.1, + drift_from_week_ago: 0.2, + ...overrides, + } +} + async function loadHealthPageModule() { return import("../page") } @@ -188,6 +209,27 @@ describe("deriveSourceStatus", () => { }) }) +describe("buildCentroidDriftTrendPoints", () => { + it("returns a sparkline across ordered centroid snapshots", async () => { + const { buildCentroidDriftTrendPoints } = await loadHealthPageModule() + + expect( + buildCentroidDriftTrendPoints([ + createTopicCentroidSnapshot({ + id: 2, + computed_at: "2026-04-29T08:00:00Z", + drift_from_previous: 0.3, + }), + createTopicCentroidSnapshot({ + id: 1, + computed_at: "2026-04-28T08:00:00Z", + drift_from_previous: 0.1, + }), + ]), + ).toBe("0.0,64.8 220.0,50.4") + }) +}) + describe("HealthPage", () => { beforeEach(() => { const defaultProject = createProject() @@ -195,12 +237,14 @@ describe("HealthPage", () => { getProjectsMock.mockReset() getProjectSourceConfigsMock.mockReset() getProjectIngestionRunsMock.mockReset() + getProjectTopicCentroidSnapshotsMock.mockReset() getProjectTopicCentroidSummaryMock.mockReset() selectProjectMock.mockReset() getProjectsMock.mockResolvedValue([defaultProject]) getProjectSourceConfigsMock.mockResolvedValue([]) getProjectIngestionRunsMock.mockResolvedValue([]) + getProjectTopicCentroidSnapshotsMock.mockResolvedValue([]) getProjectTopicCentroidSummaryMock.mockResolvedValue( createTopicCentroidSummary(), ) @@ -224,6 +268,7 @@ describe("HealthPage", () => { ).toBeInTheDocument() expect(getProjectSourceConfigsMock).not.toHaveBeenCalled() expect(getProjectIngestionRunsMock).not.toHaveBeenCalled() + expect(getProjectTopicCentroidSnapshotsMock).not.toHaveBeenCalled() expect(getProjectTopicCentroidSummaryMock).not.toHaveBeenCalled() }) @@ -238,6 +283,7 @@ describe("HealthPage", () => { ).toBeInTheDocument() expect(getProjectSourceConfigsMock).toHaveBeenCalledWith(1) expect(getProjectIngestionRunsMock).toHaveBeenCalledWith(1) + expect(getProjectTopicCentroidSnapshotsMock).toHaveBeenCalledWith(1) expect(getProjectTopicCentroidSummaryMock).toHaveBeenCalledWith(1) }) @@ -300,6 +346,23 @@ describe("HealthPage", () => { }) it("renders centroid summary cards for the selected project", async () => { + getProjectTopicCentroidSnapshotsMock.mockResolvedValue([ + createTopicCentroidSnapshot({ + id: 7, + computed_at: "2026-04-26T08:00:00Z", + drift_from_previous: 0.05, + }), + createTopicCentroidSnapshot({ + id: 8, + computed_at: "2026-04-27T08:00:00Z", + drift_from_previous: 0.1, + }), + createTopicCentroidSnapshot({ + id: 9, + computed_at: "2026-04-28T08:00:00Z", + drift_from_previous: 0.2, + }), + ]) getProjectTopicCentroidSummaryMock.mockResolvedValue( createTopicCentroidSummary({ snapshot_count: 3, @@ -325,9 +388,22 @@ describe("HealthPage", () => { expect( screen.getByText("Topic centroid observability"), ).toBeInTheDocument() - expect(screen.getByText("10.0%")).toBeInTheDocument() - expect(screen.getByText("20.0%")).toBeInTheDocument() + expect(screen.getAllByText("10.0%").length).toBeGreaterThan(0) + expect(screen.getAllByText("20.0%").length).toBeGreaterThan(0) expect(screen.getByText("Feedback 14")).toBeInTheDocument() - expect(screen.getByText("active")).toBeInTheDocument() + expect(screen.getAllByText("active").length).toBeGreaterThan(0) + expect( + screen.getByRole("link", { name: "Open centroid snapshot history" }), + ).toHaveAttribute( + "href", + "/admin/health?project=1#centroid-snapshot-history", + ) + expect( + screen.getByLabelText("Centroid drift trend"), + ).toBeInTheDocument() + expect( + screen.getByText("Centroid snapshot history"), + ).toBeInTheDocument() + expect(screen.getByText("Showing 3 of 3 snapshots")).toBeInTheDocument() }) }) diff --git a/frontend/src/app/admin/health/page.tsx b/frontend/src/app/admin/health/page.tsx index 2ec06008..91e3d45b 100644 --- a/frontend/src/app/admin/health/page.tsx +++ b/frontend/src/app/admin/health/page.tsx @@ -1,12 +1,19 @@ +import Link from "next/link" + import { AppShell } from "@/components/app-shell" import { StatusBadge } from "@/components/status-badge" import { getProjectIngestionRuns, getProjects, getProjectSourceConfigs, + getProjectTopicCentroidSnapshots, getProjectTopicCentroidSummary, } from "@/lib/api" -import type { HealthStatus, TopicCentroidObservabilitySummary } from "@/lib/types" +import type { + HealthStatus, + TopicCentroidObservabilitySummary, + TopicCentroidSnapshot, +} from "@/lib/types" import { formatDate, healthTone, selectProject } from "@/lib/view-helpers" type HealthPageProps = { @@ -79,6 +86,35 @@ export function formatDriftPercent(value: number | null) { return `${(value * 100).toFixed(1)}%` } +/** + * Build sparkline points for centroid drift across recent snapshots. + * + * @param snapshots - Persisted centroid snapshots for the selected project. + * @returns SVG polyline points spanning the recent drift history. + */ +export function buildCentroidDriftTrendPoints( + snapshots: TopicCentroidSnapshot[], +) { + if (snapshots.length <= 1) { + return "0,36 220,36" + } + + const points = snapshots + .slice() + .sort( + (left, right) => + new Date(left.computed_at).getTime() - new Date(right.computed_at).getTime(), + ) + .map((snapshot, index, orderedSnapshots) => { + const x = (index / (orderedSnapshots.length - 1)) * 220 + const drift = snapshot.drift_from_previous ?? 0 + const y = 72 - drift * 72 + return `${x.toFixed(1)},${y.toFixed(1)}` + }) + + return points.join(" ") +} + /** * Render the source-by-source ingestion health view for the selected project. * @@ -111,11 +147,22 @@ export default async function HealthPage({ searchParams }: HealthPageProps) { ) } - const [sourceConfigs, ingestionRuns, centroidSummary] = await Promise.all([ + const [sourceConfigs, ingestionRuns, centroidSummary, centroidSnapshots] = await Promise.all([ getProjectSourceConfigs(selectedProject.id), getProjectIngestionRuns(selectedProject.id), getProjectTopicCentroidSummary(selectedProject.id), + getProjectTopicCentroidSnapshots(selectedProject.id), ]) + const sortedCentroidSnapshots = centroidSnapshots + .slice() + .sort( + (left, right) => + new Date(right.computed_at).getTime() - new Date(left.computed_at).getTime(), + ) + const visibleCentroidSnapshots = sortedCentroidSnapshots.slice(0, 12) + const centroidDriftTrendPoints = buildCentroidDriftTrendPoints( + visibleCentroidSnapshots, + ) const latestRunByPlugin = new Map() for (const ingestionRun of ingestionRuns) { @@ -190,6 +237,34 @@ export default async function HealthPage({ searchParams }: HealthPageProps) { + {visibleCentroidSnapshots.length > 1 ? ( + +
+ Recent drift trend + Last {visibleCentroidSnapshots.length} snapshots +
+ + + + + ) : null} + {centroidSummary.latest_snapshot ? (
{centroidSummary.snapshot_count} snapshots @@ -207,6 +282,71 @@ export default async function HealthPage({ searchParams }: HealthPageProps) { )} +
+
+
+

+ Centroid snapshot history +

+

+ Recent centroid recomputations for this project, including feedback volume and drift between snapshots. +

+
+ + Showing {visibleCentroidSnapshots.length} of {centroidSummary.snapshot_count} snapshots + +
+ + {visibleCentroidSnapshots.length === 0 ? ( +
+ No centroid snapshot history exists for this project yet. +
+ ) : ( +
+ + + + + + + + + + + + {visibleCentroidSnapshots.map((snapshot) => ( + + + + + + + + ))} + +
ComputedStateFeedbackDrift vs previousDrift vs 7d
+ {formatDate(snapshot.computed_at)} + + + {snapshot.centroid_active ? "active" : "inactive"} + + + {snapshot.feedback_count} total + + {formatDriftPercent(snapshot.drift_from_previous)} + + {formatDriftPercent(snapshot.drift_from_week_ago)} +
+
+ )} +
+
diff --git a/frontend/src/app/admin/sources/__tests__/page.test.tsx b/frontend/src/app/admin/sources/__tests__/page.test.tsx index 192791b9..a18e518d 100644 --- a/frontend/src/app/admin/sources/__tests__/page.test.tsx +++ b/frontend/src/app/admin/sources/__tests__/page.test.tsx @@ -2,15 +2,28 @@ import { render, screen } from "@testing-library/react" import type { ReactNode } from "react" import { beforeEach, describe, expect, it, vi } from "vitest" -import type { IngestionRun, Project, SourceConfig } from "@/lib/types" +import type { + BlueskyCredentials, + IngestionRun, + IntakeAllowlistEntry, + NewsletterIntake, + Project, + SourceConfig, +} from "@/lib/types" const { + getProjectBlueskyCredentialsMock, getProjectIngestionRunsMock, + getProjectIntakeAllowlistMock, + getProjectNewsletterIntakesMock, getProjectsMock, getProjectSourceConfigsMock, selectProjectMock, } = vi.hoisted(() => ({ + getProjectBlueskyCredentialsMock: vi.fn(), getProjectIngestionRunsMock: vi.fn(), + getProjectIntakeAllowlistMock: vi.fn(), + getProjectNewsletterIntakesMock: vi.fn(), getProjectsMock: vi.fn(), getProjectSourceConfigsMock: vi.fn(), selectProjectMock: vi.fn(), @@ -49,7 +62,10 @@ vi.mock("@/components/status-badge", () => ({ })) vi.mock("@/lib/api", () => ({ + getProjectBlueskyCredentials: getProjectBlueskyCredentialsMock, getProjectIngestionRuns: getProjectIngestionRunsMock, + getProjectIntakeAllowlist: getProjectIntakeAllowlistMock, + getProjectNewsletterIntakes: getProjectNewsletterIntakesMock, getProjects: getProjectsMock, getProjectSourceConfigs: getProjectSourceConfigsMock, })) @@ -72,6 +88,13 @@ function createProject(overrides: Partial = {}): Project { group: 10, topic_description: "AI news", content_retention_days: 30, + intake_token: "intake-token-123", + intake_enabled: false, + has_bluesky_credentials: false, + bluesky_handle: "", + bluesky_is_active: false, + bluesky_last_verified_at: null, + bluesky_last_error: "", created_at: "2026-04-01T00:00:00Z", ...overrides, } @@ -108,6 +131,58 @@ function createIngestionRun( } } +function createAllowlistEntry( + overrides: Partial = {}, +): IntakeAllowlistEntry { + return { + id: 11, + project: 1, + sender_email: "newsletter@example.com", + is_confirmed: false, + confirmed_at: null, + confirmation_token: "confirm-token-123", + created_at: "2026-04-28T08:00:00Z", + ...overrides, + } +} + +function createNewsletterIntake( + overrides: Partial = {}, +): NewsletterIntake { + return { + id: 31, + project: 1, + sender_email: "newsletter@example.com", + subject: "Morning digest", + received_at: "2026-04-29T08:15:00Z", + raw_html: "", + raw_text: "Top story https://example.com/post", + message_id: "msg-31", + status: "pending", + extraction_result: null, + error_message: "", + ...overrides, + } +} + +function createBlueskyCredentials( + overrides: Partial = {}, +): BlueskyCredentials { + return { + id: 6, + project: 1, + handle: "project.bsky.social", + pds_url: "", + is_active: true, + has_stored_credential: true, + last_verified_at: "2026-04-29T10:00:00Z", + last_error: "", + created_at: "2026-04-29T09:00:00Z", + updated_at: "2026-04-29T10:00:00Z", + ...overrides, + } +} + async function loadSourcesPageModule() { return import("../page") } @@ -144,18 +219,41 @@ describe("buildLatestRunByPlugin", () => { }) }) +describe("filterNewsletterIntakes", () => { + it("filters newsletter intake rows by status and sender", async () => { + const { filterNewsletterIntakes } = await loadSourcesPageModule() + + const filtered = filterNewsletterIntakes( + [ + createNewsletterIntake({ id: 1, status: "pending", sender_email: "first@example.com" }), + createNewsletterIntake({ id: 2, status: "extracted", sender_email: "second@example.com" }), + ], + { status: "extracted", sender: "second" }, + ) + + expect(filtered).toHaveLength(1) + expect(filtered[0].id).toBe(2) + }) +}) + describe("SourcesPage", () => { beforeEach(() => { const defaultProject = createProject() + getProjectBlueskyCredentialsMock.mockReset() getProjectsMock.mockReset() getProjectSourceConfigsMock.mockReset() getProjectIngestionRunsMock.mockReset() + getProjectIntakeAllowlistMock.mockReset() + getProjectNewsletterIntakesMock.mockReset() selectProjectMock.mockReset() + getProjectBlueskyCredentialsMock.mockResolvedValue([]) getProjectsMock.mockResolvedValue([defaultProject]) getProjectSourceConfigsMock.mockResolvedValue([]) getProjectIngestionRunsMock.mockResolvedValue([]) + getProjectIntakeAllowlistMock.mockResolvedValue([]) + getProjectNewsletterIntakesMock.mockResolvedValue([]) selectProjectMock.mockImplementation((projects: Project[]) => { return projects[0] ?? null }) @@ -176,6 +274,9 @@ describe("SourcesPage", () => { ).toBeInTheDocument() expect(getProjectSourceConfigsMock).not.toHaveBeenCalled() expect(getProjectIngestionRunsMock).not.toHaveBeenCalled() + expect(getProjectBlueskyCredentialsMock).not.toHaveBeenCalled() + expect(getProjectIntakeAllowlistMock).not.toHaveBeenCalled() + expect(getProjectNewsletterIntakesMock).not.toHaveBeenCalled() }) it("renders flash messages from the search params", async () => { @@ -203,8 +304,123 @@ describe("SourcesPage", () => { expect( screen.getByText("No source configurations exist for this project yet."), ).toBeInTheDocument() + expect(screen.getByDisplayValue("intake-token-123")).toBeInTheDocument() + expect( + screen.getByRole("button", { name: "Verify credentials" }), + ).toBeDisabled() + expect( + screen.getByRole("button", { name: "Rotate token" }), + ).toBeInTheDocument() expect(getProjectSourceConfigsMock).toHaveBeenCalledWith(1) expect(getProjectIngestionRunsMock).toHaveBeenCalledWith(1) + expect(getProjectBlueskyCredentialsMock).toHaveBeenCalledWith(1) + expect(getProjectIntakeAllowlistMock).toHaveBeenCalledWith(1) + expect(getProjectNewsletterIntakesMock).toHaveBeenCalledWith(1) + }) + + it("renders allowlist management and recent intake history", async () => { + getProjectIntakeAllowlistMock.mockResolvedValue([ + createAllowlistEntry({ + id: 1, + is_confirmed: true, + confirmed_at: "2026-04-29T09:00:00Z", + }), + createAllowlistEntry({ + id: 2, + sender_email: "pending@example.com", + }), + ]) + getProjectNewsletterIntakesMock.mockResolvedValue([ + createNewsletterIntake({ + id: 1, + status: "extracted", + extraction_result: { + method: "heuristic", + items: [ + { + title: "Story one", + url: "https://example.com/story-one", + excerpt: "First story", + position: 1, + }, + ], + }, + }), + createNewsletterIntake({ + id: 2, + subject: "Follow-up digest", + sender_email: "pending@example.com", + status: "pending", + }), + ]) + + await renderSourcesPage({ project: "1" }) + + expect(screen.getByText("Sender allowlist")).toBeInTheDocument() + expect(screen.getAllByText("newsletter@example.com")).toHaveLength(3) + expect(screen.getAllByText("pending@example.com")).toHaveLength(2) + expect(screen.getByText("Recent newsletter intake")).toBeInTheDocument() + expect(screen.getAllByText("Story one")).toHaveLength(2) + expect(screen.getByText("Follow-up digest")).toBeInTheDocument() + expect(screen.getAllByRole("link", { name: "Open details" })).toHaveLength(2) + expect(screen.getByText("Selected intake")).toBeInTheDocument() + }) + + it("applies intake filters from the search params", async () => { + getProjectNewsletterIntakesMock.mockResolvedValue([ + createNewsletterIntake({ id: 1, sender_email: "first@example.com", status: "pending" }), + createNewsletterIntake({ + id: 2, + sender_email: "editor@example.com", + status: "extracted", + subject: "Filtered digest", + }), + ]) + + await renderSourcesPage({ + project: "1", + intakeStatus: "extracted", + intakeSender: "editor", + intakeId: "2", + }) + + expect(screen.getByDisplayValue("editor")).toBeInTheDocument() + expect(screen.getAllByText("Filtered digest")).toHaveLength(2) + expect(screen.queryByText("Morning digest")).not.toBeInTheDocument() + }) + + it("renders intake controls and Bluesky verification state from the selected project", async () => { + const selectedProject = createProject({ + id: 3, + intake_enabled: true, + intake_token: "intake-token-xyz", + has_bluesky_credentials: true, + bluesky_handle: "project.bsky.social", + bluesky_is_active: true, + bluesky_last_verified_at: "2026-04-29T10:00:00Z", + }) + + getProjectsMock.mockResolvedValue([selectedProject]) + selectProjectMock.mockReturnValue(selectedProject) + getProjectBlueskyCredentialsMock.mockResolvedValue([ + createBlueskyCredentials({ project: 3 }), + ]) + + await renderSourcesPage({ project: "3" }) + + expect(screen.getByText("Project intake settings")).toBeInTheDocument() + expect(screen.getByDisplayValue("intake-token-xyz")).toBeInTheDocument() + expect( + screen.getByDisplayValue("intake+intake-token-xyz@inbox.example.com"), + ).toBeInTheDocument() + expect(screen.getByText("project.bsky.social")).toBeInTheDocument() + expect(screen.getByText("verified")).toBeInTheDocument() + expect( + screen.getByRole("button", { name: "Verify credentials" }), + ).toBeEnabled() + expect( + screen.getByRole("button", { name: "Update credentials" }), + ).toBeInTheDocument() }) it("renders source cards with badge tones and the latest run summary", async () => { @@ -258,11 +474,21 @@ describe("SourcesPage", () => { expect(screen.getByText("Rate limited")).toBeInTheDocument() const badges = screen.getAllByTestId("status-badge") - expect(badges).toHaveLength(2) - expect(badges[0]).toHaveAttribute("data-tone", "positive") - expect(badges[0]).toHaveTextContent("active") - expect(badges[1]).toHaveAttribute("data-tone", "neutral") - expect(badges[1]).toHaveTextContent("disabled") + expect(badges).toHaveLength(4) + expect( + badges.some( + (badge) => + badge.getAttribute("data-tone") === "neutral" && + badge.textContent === "disabled", + ), + ).toBe(true) + expect( + badges.some( + (badge) => + badge.getAttribute("data-tone") === "positive" && + badge.textContent === "active", + ), + ).toBe(true) }) it("shows fallback latest-run text when a source has no ingestion history", async () => { diff --git a/frontend/src/app/admin/sources/page.tsx b/frontend/src/app/admin/sources/page.tsx index 69ad525e..365823cb 100644 --- a/frontend/src/app/admin/sources/page.tsx +++ b/frontend/src/app/admin/sources/page.tsx @@ -1,10 +1,15 @@ import { AppShell } from "@/components/app-shell" +import { CopyButton } from "@/components/copy-button" import { StatusBadge } from "@/components/status-badge" import { + getProjectBlueskyCredentials, getProjectIngestionRuns, + getProjectIntakeAllowlist, + getProjectNewsletterIntakes, getProjects, getProjectSourceConfigs, } from "@/lib/api" +import type { BlueskyCredentials, NewsletterIntake, Project } from "@/lib/types" import { formatDate, getErrorMessage, @@ -12,6 +17,49 @@ import { selectProject, } from "@/lib/view-helpers" +type BlueskyVerificationState = { + label: string + tone: "positive" | "warning" | "negative" | "neutral" +} + +/** + * Build the documented newsletter intake address pattern for one project token. + * + * The backend stores only the per-project token today, not the mail-provider domain. + * This helper renders the documented address pattern so editors can copy the project + * token and see how it is expected to be used with the inbound mailbox domain. + * + * @param intakeToken - The stable per-project intake token. + * @returns The documented intake address pattern. + */ +export function buildIntakeAddressTemplate(intakeToken: string) { + return `intake+${intakeToken || ""}@inbox.example.com` +} + +/** + * Derive the current Bluesky verification badge state for the selected project. + * + * @param project - Project record returned from the backend. + * @returns A badge label and semantic tone describing the stored credential state. + */ +export function deriveBlueskyVerificationState( + project: Project, +): BlueskyVerificationState { + if (!project.has_bluesky_credentials) { + return { label: "not configured", tone: "neutral" } + } + + if (project.bluesky_last_error) { + return { label: "verification failed", tone: "negative" } + } + + if (project.bluesky_last_verified_at) { + return { label: "verified", tone: "positive" } + } + + return { label: "needs verification", tone: "warning" } +} + type SourcesPageProps = { searchParams: Promise> } @@ -38,6 +86,55 @@ export function buildLatestRunByPlugin( return latestRunByPlugin } +/** + * Build a concise newsletter-intake preview from persisted extraction data. + * + * @param intake - One persisted newsletter intake row. + * @returns A short human-readable preview for the intake history card. + */ +export function buildNewsletterIntakePreview(intake: NewsletterIntake) { + const extractedItems = intake.extraction_result?.items ?? [] + if (extractedItems.length > 0) { + return extractedItems + .slice(0, 2) + .map((item) => item.title || item.url) + .join("; ") + } + + if (intake.error_message) { + return intake.error_message + } + + return intake.raw_text.slice(0, 160) || "No preview available yet." +} + +/** + * Filter newsletter intake rows using URL-driven sender and status criteria. + * + * @param newsletterIntakes - Full newsletter intake list for the selected project. + * @param filters - Filter values read from the sources page search params. + * @returns The filtered intake rows. + */ +export function filterNewsletterIntakes( + newsletterIntakes: NewsletterIntake[], + filters: { status: string; sender: string }, +) { + const normalizedSender = filters.sender.trim().toLowerCase() + + return newsletterIntakes.filter((intake) => { + if (filters.status && intake.status !== filters.status) { + return false + } + if ( + normalizedSender && + !intake.sender_email.toLowerCase().includes(normalizedSender) + ) { + return false + } + return true + }) +} + /** * Render the source-configuration admin page for the selected project. * @@ -70,11 +167,44 @@ export default async function SourcesPage({ searchParams }: SourcesPageProps) { ) } - const [sourceConfigs, ingestionRuns] = await Promise.all([ + const [ + sourceConfigs, + ingestionRuns, + intakeAllowlist, + newsletterIntakes, + blueskyCredentials, + ] = await Promise.all([ getProjectSourceConfigs(selectedProject.id), getProjectIngestionRuns(selectedProject.id), + getProjectIntakeAllowlist(selectedProject.id), + getProjectNewsletterIntakes(selectedProject.id), + getProjectBlueskyCredentials(selectedProject.id), ]) const latestRunByPlugin = buildLatestRunByPlugin(ingestionRuns) + const blueskyVerificationState = deriveBlueskyVerificationState(selectedProject) + const intakeAddressTemplate = buildIntakeAddressTemplate( + selectedProject.intake_token ?? "", + ) + const sortedSourceConfigs = sourceConfigs + .slice() + .sort((left, right) => left.plugin_name.localeCompare(right.plugin_name)) + const intakeStatusFilter = String(resolvedSearchParams.intakeStatus || "") + const intakeSenderFilter = String(resolvedSearchParams.intakeSender || "") + const selectedIntakeId = Number.parseInt( + String(resolvedSearchParams.intakeId || "0"), + 10, + ) + const filteredNewsletterIntakes = filterNewsletterIntakes(newsletterIntakes, { + status: intakeStatusFilter, + sender: intakeSenderFilter, + }) + const recentNewsletterIntakes = filteredNewsletterIntakes.slice(0, 6) + const selectedIntake = + newsletterIntakes.find((intake) => intake.id === selectedIntakeId) ?? + recentNewsletterIntakes[0] ?? + null + const currentBlueskyCredentials: BlueskyCredentials | null = + blueskyCredentials[0] ?? null const errorMessage = getErrorMessage(resolvedSearchParams) const successMessage = getSuccessMessage(resolvedSearchParams) @@ -82,7 +212,7 @@ export default async function SourcesPage({ searchParams }: SourcesPageProps) { return ( @@ -94,66 +224,502 @@ export default async function SourcesPage({ searchParams }: SourcesPageProps) { ) : null}
-
-

Add source

-
- - - -