diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 96debfe2..035c0793 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -36,7 +36,10 @@ jobs: cache-dependency-path: frontend/package-lock.json - name: Install just - uses: extractions/setup-just@v4 + uses: extractions/setup-crate@v2 + with: + repo: casey/just@1.50.0 + github-token: ${{ github.token }} - name: Install dependencies run: just install diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 78a27a4a..7a66ef3f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,7 +36,10 @@ jobs: cache-dependency-path: frontend/package-lock.json - name: Install just - uses: extractions/setup-just@v4 + uses: extractions/setup-crate@v2 + with: + repo: casey/just@1.50.0 + github-token: ${{ github.token }} - name: Install dependencies run: just install diff --git a/README.md b/README.md index c9b3a6e8..c34487b0 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,14 @@ An AI-powered content curation platform for technically-oriented newsletters. Newsletter Maker ingests content from dozens of sources, builds authority models of people and companies in a domain, and surfaces the most relevant articles, trends, and themes for each edition — so editors spend their time writing, not searching. -The system is multi-tenant: each newsletter has its own tracked entities, relevance model, and content pipeline. Designed for non-technical editors who don't know what a vector database is and don't need to. +The system is organized into projects: each newsletter project has its own tracked entities, relevance model, and content pipeline. Projects are assigned to Django groups so editorial access can be shared cleanly. Designed for non-technical editors who don't know what a vector database is and don't need to. ## What This Does That Existing Tools Don't Tools like Feedly, UpContent, and ContentStudio handle parts of the content curation problem. Newsletter Maker combines several capabilities none of them offer: - **Authority scoring from newsletter cross-referencing.** By ingesting peer newsletters, the system builds an authority model based on who real editors actually link to — a human-curated endorsement signal no existing tool provides. -- **Per-editor relevance training.** Upvote/downvote feedback trains a personalized relevance model per tenant. The tool learns what *you* consider valuable. +- **Per-project relevance training.** Upvote/downvote feedback trains a personalized relevance model per project. The tool learns what each editorial project considers valuable. - **Unified entity model.** A person's blog, LinkedIn, Bluesky, GitHub, and conference talks are linked into a single profile with an authority score — a holistic view of who matters in a space. - **Competitive intelligence.** "These 3 peer newsletters all covered topic X this week, but you haven't." A natural output of newsletter ingestion that no curation tool provides. - **Historical trend analysis.** Not just what's trending now, but trajectories over weeks. Content is retained indefinitely for long-term pattern detection. @@ -102,8 +102,9 @@ python3 -m pip install -r requirements.txt 1. Run `just dev` to start Django, Celery, Postgres, Redis, Qdrant, and Nginx. On the first run Docker builds the app image automatically. After that, `just dev` reuses the existing image so normal restarts are fast. If `.env` is missing, the `just` command copies `.env.example` automatically. 2. Run `just build` after changing `requirements.txt` or `docker/web/Dockerfile`. -3. Update `.env` with non-default secrets before using the stack outside local development. The example file uses SQLite and localhost URLs so host-side `manage.py` commands work even without Docker. -4. Open `http://localhost:8080/healthz/` for a liveness check and `http://localhost:8080/admin/` for Django admin. +3. For a fully fresh local stack after schema changes, run `just reset-volumes` before starting the containers again. This drops the Docker-backed Postgres, Redis, and Qdrant state so regenerated migrations apply cleanly. +4. Update `.env` with non-default secrets before using the stack outside local development. The example file uses SQLite and localhost URLs so host-side `manage.py` commands work even without Docker. +5. Open `http://localhost:8080/healthz/` for a liveness check and `http://localhost:8080/admin/` for Django admin. Use `just seed` after the stack is up if you want the demo project and sample content. For host-based development without Docker, install `requirements.txt`, then use `python3 manage.py migrate` and `python3 manage.py runserver`. The default `.env.example` is host-safe; Docker Compose overrides the service URLs inside containers. @@ -155,7 +156,7 @@ Use these commands to backfill or refresh embeddings for existing content: ```bash just embed-all -just embed-tenant 1 +just embed-project 1 python3 manage.py sync_embeddings --content-id 42 python3 manage.py sync_embeddings --references-only ``` diff --git a/core/admin.py b/core/admin.py index 9b16c9fb..25b13188 100644 --- a/core/admin.py +++ b/core/admin.py @@ -12,60 +12,60 @@ Content, Entity, IngestionRun, + Project, + ProjectConfig, ReviewQueue, SkillResult, SourceConfig, - Tenant, - TenantConfig, UserFeedback, ) from core.plugins import get_plugin_for_source_config, validate_plugin_config from core.tasks import process_content -@admin.register(Tenant) -class TenantAdmin(ExportActionMixin, admin.ModelAdmin): - list_display = ("name", "user", "content_retention_days", "created_at") +@admin.register(Project) +class ProjectAdmin(ExportActionMixin, admin.ModelAdmin): + list_display = ("name", "group", "content_retention_days", "created_at") - # Better navigation - date_hierarchy = "created_at" - list_filter = ("created_at",) + # Better navigation + date_hierarchy = "created_at" + list_filter = ("created_at",) - # Faster searching - search_fields = ("name", "user__username", "user__email") + # Faster searching + search_fields = ("name", "group__name") - # Performance for large user lists - autocomplete_fields = ("user",) + # Performance for large user lists + autocomplete_fields = ("group",) - # Quick editing - list_editable = ("content_retention_days",) + # Quick editing + list_editable = ("content_retention_days",) -@admin.register(TenantConfig) -class TenantConfigAdmin(admin.ModelAdmin): - list_display = ("tenant", "upvote_authority_weight", "downvote_authority_weight", "authority_decay_rate") +@admin.register(ProjectConfig) +class ProjectConfigAdmin(admin.ModelAdmin): + list_display = ("project", "upvote_authority_weight", "downvote_authority_weight", "authority_decay_rate") @admin.register(Entity) class EntityAdmin(admin.ModelAdmin): - # Replace 'authority_score' with your new method name - list_display = ("name", "tenant", "type", "colored_score", "created_at") - - @admin.display(description="Authority Score", ordering="authority_score") - def colored_score(self, obj): - # Choose a color based on the value - if obj.authority_score >= 80: - color = "green" - elif obj.authority_score >= 50: - color = "orange" - else: - color = "red" - - return format_html( - '{}', - color, - obj.authority_score, - ) + # Replace 'authority_score' with your new method name + list_display = ("name", "project", "type", "colored_score", "created_at") + + @admin.display(description="Authority Score", ordering="authority_score") + def colored_score(self, obj): + # Choose a color based on the value + if obj.authority_score >= 80: + color = "green" + elif obj.authority_score >= 50: + color = "orange" + else: + color = "red" + + return format_html( + '{}', + color, + obj.authority_score, + ) class HighValueFilter(admin.SimpleListFilter): @@ -91,14 +91,14 @@ class ContentAdmin(admin.ModelAdmin): "is_reference", "preview_content", "source_plugin", - "tenant", + "project", "title", "view_trace", ) list_editable = ("is_reference", "is_active") list_filter = ( HighValueFilter, - ("tenant", admin.RelatedOnlyFieldListFilter), + ("project", admin.RelatedOnlyFieldListFilter), "source_plugin", "is_active" ) @@ -159,7 +159,7 @@ def view_trace(self, obj): run_id=trace_id, skill_name=latest_skill_result.skill_name, skill_result_id=latest_skill_result.id, - tenant_id=obj.tenant_id, + project_id=obj.project_id, trace_id=trace_id, ) @@ -233,13 +233,13 @@ class SkillResultAdmin(ModelAdmin): "model_used", "created_at", ) - list_filter = ("status", "skill_name", "tenant", "model_used") + list_filter = ("status", "skill_name", "project", "model_used") search_fields = ("skill_name", "content__title", "model_used", "error_message") actions = ["retry_selected_skills"] readonly_fields = ("pretty_result_data", "latency_ms", "created_at", "superseded_by") fieldsets = ( ("Execution Details", { - "fields": ("skill_name", "content", "tenant", "status", "model_used") + "fields": ("skill_name", "content", "project", "status", "model_used") }), ("AI Output", { "fields": ("pretty_result_data", "error_message"), @@ -334,11 +334,11 @@ class UserFeedbackAdmin(ModelAdmin): "display_feedback", "get_content_title", "get_ai_score", - "tenant", + "project", "user", "created_at" ) - list_filter = ("feedback_type", ("tenant", admin.RelatedOnlyFieldListFilter)) + list_filter = ("feedback_type", ("project", admin.RelatedOnlyFieldListFilter)) search_fields = ("content__title", "user__email", "user__username") @admin.display(description="Type") @@ -387,18 +387,18 @@ def changelist_view(self, request, extra_context=None): class IngestionRunAdmin(ModelAdmin): list_display = ( "plugin_name", - "tenant", + "project", "display_status", "display_efficiency", "display_duration", "started_at", ) - list_filter = ("plugin_name", "status", ("tenant", admin.RelatedOnlyFieldListFilter)) - search_fields = ("plugin_name", "error_message", "tenant__name") + list_filter = ("plugin_name", "status", ("project", admin.RelatedOnlyFieldListFilter)) + search_fields = ("plugin_name", "error_message", "project__name") readonly_fields = ("display_duration", "started_at", "completed_at") fieldsets = ( ("Run Info", { - "fields": ("plugin_name", "tenant", "status") + "fields": ("plugin_name", "project", "status") }), ("Data Metrics", { "fields": ("items_fetched", "items_ingested", "display_efficiency") @@ -468,19 +468,19 @@ def changelist_view(self, request, extra_context=None): class SourceConfigAdmin(ModelAdmin): list_display = ( "plugin_name", - "tenant", + "project", "display_health", "is_active", "last_fetched_at", ) - list_filter = ("is_active", "plugin_name", ("tenant", admin.RelatedOnlyFieldListFilter)) + list_filter = ("is_active", "plugin_name", ("project", admin.RelatedOnlyFieldListFilter)) list_editable = ("is_active",) - search_fields = ("plugin_name", "tenant__name") + search_fields = ("plugin_name", "project__name") actions = ["test_source_connection"] readonly_fields = ("last_fetched_at", "pretty_config") fieldsets = ( ("Core Settings", { - "fields": ("plugin_name", "tenant", "is_active") + "fields": ("plugin_name", "project", "is_active") }), ("Configuration", { "fields": ("pretty_config", "config"), @@ -519,7 +519,7 @@ def test_source_connection(self, request, queryset): healthy_sources = [] failed_sources = [] - for source_config in queryset.select_related("tenant"): + for source_config in queryset.select_related("project"): try: source_config.config = validate_plugin_config( source_config.plugin_name, @@ -573,14 +573,14 @@ def changelist_view(self, request, extra_context=None): class ReviewQueueAdmin(ModelAdmin): list_display = ( "get_content_title", - "tenant", + "project", "reason", "display_confidence", "resolved", "resolution", "created_at", ) - list_filter = ("resolved", "reason", ("tenant", admin.RelatedOnlyFieldListFilter)) + list_filter = ("resolved", "reason", ("project", admin.RelatedOnlyFieldListFilter)) list_editable = ("resolved", "resolution") actions = ["mark_as_approved", "mark_as_rejected"] diff --git a/core/api.py b/core/api.py index 71caa405..4c4ccc4c 100644 --- a/core/api.py +++ b/core/api.py @@ -17,11 +17,11 @@ Content, Entity, IngestionRun, + Project, + ProjectConfig, ReviewQueue, SkillResult, SourceConfig, - Tenant, - TenantConfig, UserFeedback, ) from core.pipeline import ( @@ -35,20 +35,20 @@ ContentSerializer, EntitySerializer, IngestionRunSerializer, + ProjectConfigSerializer, + ProjectSerializer, ReviewQueueSerializer, SkillResultSerializer, SourceConfigSerializer, - TenantConfigSerializer, - TenantSerializer, UserFeedbackSerializer, ) from core.tasks import queue_content_skill -TENANT_ID_PARAMETER = OpenApiParameter( - name="tenant_id", +PROJECT_ID_PARAMETER = OpenApiParameter( + name="project_id", type=int, location=OpenApiParameter.PATH, - description="The unique ID of the tenant that owns this nested resource.", + description="The unique ID of the project that owns this nested resource.", ) SKILL_NAME_PARAMETER = OpenApiParameter( @@ -61,22 +61,23 @@ ), ) -TENANT_CREATE_REQUEST_EXAMPLE = OpenApiExample( - "Create Tenant Request", +PROJECT_CREATE_REQUEST_EXAMPLE = OpenApiExample( + "Create Project Request", value={ "name": "AI Weekly", + "group": 3, "topic_description": "Coverage of developer tools, model releases, and applied AI workflows.", "content_retention_days": 180, }, request_only=True, ) -TENANT_RESPONSE_EXAMPLE = OpenApiExample( - "Tenant Response", +PROJECT_RESPONSE_EXAMPLE = OpenApiExample( + "Project Response", value={ "id": 1, "name": "AI Weekly", - "user": 7, + "group": 3, "topic_description": "Coverage of developer tools, model releases, and applied AI workflows.", "content_retention_days": 180, "created_at": "2026-04-26T12:00:00Z", @@ -114,7 +115,7 @@ "Source Configuration Response", value={ "id": 12, - "tenant": 1, + "project": 1, "plugin_name": "rss", "config": { "feed_url": "https://example.com/feed.xml", @@ -147,7 +148,7 @@ "Content Response", value={ "id": 44, - "tenant": 1, + "project": 1, "url": "https://example.com/posts/agent-memory-patterns", "title": "Practical Agent Memory Patterns", "author": "Jane Doe", @@ -170,7 +171,7 @@ value={ "id": 91, "content": 44, - "tenant": 1, + "project": 1, "skill_name": "relevance_classifier", "status": "completed", "result_data": { @@ -299,7 +300,7 @@ def build_crud_action_overrides( return overrides -def document_user_owned_viewset( +def document_group_access_viewset( resource_plural: str, resource_singular: str, create_description: str, @@ -323,12 +324,12 @@ def schema(action: str, **kwargs): list=schema( "list", summary=f"List {resource_plural}", - description=f"Return all {resource_plural} owned by the authenticated user.", + description=f"Return all {resource_plural} available to the authenticated user through group membership.", ), retrieve=schema( "retrieve", summary=f"Get {resource_singular}", - description=f"Return a single {resource_singular} owned by the authenticated user.", + description=f"Return a single {resource_singular} available to the authenticated user through group membership.", ), create=schema( "create", @@ -338,29 +339,29 @@ def schema(action: str, **kwargs): update=schema( "update", summary=f"Replace {resource_singular}", - description=f"Replace an existing {resource_singular} owned by the authenticated user.", + description=f"Replace an existing {resource_singular} available to the authenticated user through group membership.", ), partial_update=schema( "partial_update", summary=f"Update {resource_singular}", - description=f"Update one or more fields on an existing {resource_singular} owned by the authenticated user.", + description=f"Update one or more fields on an existing {resource_singular} available to the authenticated user through group membership.", ), destroy=schema( "destroy", summary=f"Delete {resource_singular}", - description=f"Delete an existing {resource_singular} owned by the authenticated user.", + description=f"Delete an existing {resource_singular} available to the authenticated user through group membership.", ), ) -def document_tenant_owned_viewset( +def document_project_owned_viewset( resource_plural: str, resource_singular: str, create_description: str, tag: str, action_overrides: dict[str, dict] | None = None, ): - parameters = [TENANT_ID_PARAMETER] + parameters = [PROJECT_ID_PARAMETER] action_overrides = action_overrides or {} def schema(action: str, **kwargs): @@ -378,13 +379,13 @@ def schema(action: str, **kwargs): list=schema( "list", summary=f"List {resource_plural}", - description=f"Return all {resource_plural} for the selected tenant.", + description=f"Return all {resource_plural} for the selected project.", parameters=parameters, ), retrieve=schema( "retrieve", summary=f"Get {resource_singular}", - description=f"Return a single {resource_singular} for the selected tenant.", + description=f"Return a single {resource_singular} for the selected project.", parameters=parameters, ), create=schema( @@ -396,127 +397,123 @@ def schema(action: str, **kwargs): update=schema( "update", summary=f"Replace {resource_singular}", - description=f"Replace an existing {resource_singular} for the selected tenant.", + description=f"Replace an existing {resource_singular} for the selected project.", parameters=parameters, ), partial_update=schema( "partial_update", summary=f"Update {resource_singular}", - description=f"Update one or more fields on an existing {resource_singular} for the selected tenant.", + description=f"Update one or more fields on an existing {resource_singular} for the selected project.", parameters=parameters, ), destroy=schema( "destroy", summary=f"Delete {resource_singular}", - description=f"Delete an existing {resource_singular} for the selected tenant.", + description=f"Delete an existing {resource_singular} for the selected project.", parameters=parameters, ), ) -class TenantOwnedQuerysetMixin: +class ProjectOwnedQuerysetMixin: queryset: Any = None - def get_tenant(self): - tenant_id = self.kwargs.get("tenant_id") - if tenant_id is None: - raise AssertionError("tenant_id must be present in nested tenant-scoped routes") + def get_project(self): + project_id = self.kwargs.get("project_id") + if project_id is None: + raise AssertionError("project_id must be present in nested project-scoped routes") try: - return Tenant.objects.get(pk=tenant_id, user=self.request.user) - except Tenant.DoesNotExist as exc: - raise NotFound("Tenant not found.") from exc + return Project.objects.get(pk=project_id, group__user=self.request.user) + except Project.DoesNotExist as exc: + raise NotFound("Project not found.") from exc def get_queryset(self): queryset = self.queryset if queryset is None: - raise AssertionError("queryset must be set on tenant-scoped viewsets") - return queryset.filter(tenant=self.get_tenant()) + raise AssertionError("queryset must be set on project-scoped viewsets") + return queryset.filter(project=self.get_project()) def get_serializer_context(self): context = super().get_serializer_context() - context["tenant"] = self.get_tenant() + context["project"] = self.get_project() return context def perform_create(self, serializer): - serializer.save(tenant=self.get_tenant()) + serializer.save(project=self.get_project()) - -@document_user_owned_viewset( - resource_plural="tenants", - resource_singular="tenant", - create_description="Create a new tenant for the authenticated user. The requesting user is attached automatically.", - tag="Tenant Management", +@document_group_access_viewset( + resource_plural="projects", + resource_singular="project", + create_description="Create a new project for one of the authenticated user's groups.", + tag="Project Management", action_overrides=build_crud_action_overrides( - TenantSerializer, - resource_plural="tenants owned by the authenticated user", - resource_singular="tenant", - create_examples=[TENANT_CREATE_REQUEST_EXAMPLE, TENANT_RESPONSE_EXAMPLE], - create_response_examples=[TENANT_RESPONSE_EXAMPLE], - retrieve_examples=[TENANT_RESPONSE_EXAMPLE], + ProjectSerializer, + resource_plural="projects available to the authenticated user", + resource_singular="project", + create_examples=[PROJECT_CREATE_REQUEST_EXAMPLE, PROJECT_RESPONSE_EXAMPLE], + create_response_examples=[PROJECT_RESPONSE_EXAMPLE], + retrieve_examples=[PROJECT_RESPONSE_EXAMPLE], ), ) -class TenantViewSet(viewsets.ModelViewSet): - serializer_class = TenantSerializer - queryset = Tenant.objects.select_related("user") +class ProjectViewSet(viewsets.ModelViewSet): + serializer_class = ProjectSerializer + queryset = Project.objects.select_related("group") lookup_url_kwarg = "id" def get_queryset(self): - return self.queryset.filter(user=self.request.user) - - def perform_create(self, serializer): - serializer.save(user=self.request.user) + return self.queryset.filter(group__user=self.request.user).distinct() -@document_tenant_owned_viewset( - resource_plural="tenant configurations", - resource_singular="tenant configuration", - create_description="Create a new tenant configuration record for the selected tenant, including authority weighting and decay settings.", - tag="Tenant Management", +@document_project_owned_viewset( + resource_plural="project configurations", + resource_singular="project configuration", + create_description="Create a new project configuration record for the selected project, including authority weighting and decay settings.", + tag="Project Management", action_overrides=build_crud_action_overrides( - TenantConfigSerializer, - resource_plural="tenant configurations for the selected tenant", - resource_singular="tenant configuration", + ProjectConfigSerializer, + resource_plural="project configurations for the selected project", + resource_singular="project configuration", ), ) -class TenantConfigViewSet(TenantOwnedQuerysetMixin, viewsets.ModelViewSet): - serializer_class = TenantConfigSerializer - queryset = TenantConfig.objects.select_related("tenant") +class ProjectConfigViewSet(ProjectOwnedQuerysetMixin, viewsets.ModelViewSet): + serializer_class = ProjectConfigSerializer + queryset = ProjectConfig.objects.select_related("project") -@document_tenant_owned_viewset( +@document_project_owned_viewset( resource_plural="entities", resource_singular="entity", - create_description="Create a new tracked entity for the selected tenant, such as a company, person, or project.", + create_description="Create a new tracked entity for the selected project, such as a company, person, or organization.", tag="Entity Catalog", action_overrides=build_crud_action_overrides( EntitySerializer, - resource_plural="entities for the selected tenant", + resource_plural="entities for the selected project", resource_singular="entity", ), ) -class EntityViewSet(TenantOwnedQuerysetMixin, viewsets.ModelViewSet): +class EntityViewSet(ProjectOwnedQuerysetMixin, viewsets.ModelViewSet): serializer_class = EntitySerializer - queryset = Entity.objects.select_related("tenant") + queryset = Entity.objects.select_related("project") -@document_tenant_owned_viewset( +@document_project_owned_viewset( resource_plural="content items", resource_singular="content item", - create_description="Create a new content item for the selected tenant. Any related entity must belong to the same tenant.", + create_description="Create a new content item for the selected project. Any related entity must belong to the same project.", tag="Content Library", action_overrides=build_crud_action_overrides( ContentSerializer, - resource_plural="content items for the selected tenant", + resource_plural="content items for the selected project", resource_singular="content item", create_examples=[CONTENT_CREATE_REQUEST_EXAMPLE, CONTENT_RESPONSE_EXAMPLE], create_response_examples=[CONTENT_RESPONSE_EXAMPLE], retrieve_examples=[CONTENT_RESPONSE_EXAMPLE], ), ) -class ContentViewSet(TenantOwnedQuerysetMixin, viewsets.ModelViewSet): +class ContentViewSet(ProjectOwnedQuerysetMixin, viewsets.ModelViewSet): serializer_class = ContentSerializer - queryset = Content.objects.select_related("tenant", "entity") + queryset = Content.objects.select_related("project", "entity") @extend_schema( summary="Run content skill", @@ -525,7 +522,7 @@ class ContentViewSet(TenantOwnedQuerysetMixin, viewsets.ModelViewSet): "Supported skill names are content_classification, relevance_scoring, summarization, and find_related." ), tags=["AI Processing"], - parameters=[TENANT_ID_PARAMETER, SKILL_NAME_PARAMETER], + parameters=[PROJECT_ID_PARAMETER, SKILL_NAME_PARAMETER], request=None, responses={ 201: SkillResultSerializer, @@ -562,66 +559,66 @@ def run_skill(self, request, *args, **kwargs): return Response(serializer.data, status=status.HTTP_201_CREATED) -@document_tenant_owned_viewset( +@document_project_owned_viewset( resource_plural="skill results", resource_singular="skill result", - create_description="Create a new skill result for tenant content. The referenced content must belong to the selected tenant.", + create_description="Create a new skill result for project content. The referenced content must belong to the selected project.", tag="AI Processing", action_overrides=build_crud_action_overrides( SkillResultSerializer, - resource_plural="skill results for the selected tenant", + resource_plural="skill results for the selected project", resource_singular="skill result", retrieve_examples=[SKILL_RESULT_RESPONSE_EXAMPLE], ), ) -class SkillResultViewSet(TenantOwnedQuerysetMixin, viewsets.ModelViewSet): +class SkillResultViewSet(ProjectOwnedQuerysetMixin, viewsets.ModelViewSet): serializer_class = SkillResultSerializer - queryset = SkillResult.objects.select_related("content", "tenant", "superseded_by") + queryset = SkillResult.objects.select_related("content", "project", "superseded_by") -@document_tenant_owned_viewset( +@document_project_owned_viewset( resource_plural="user feedback entries", resource_singular="user feedback entry", - create_description="Create a new feedback entry for content in the selected tenant. The authenticated user is recorded automatically.", + create_description="Create a new feedback entry for content in the selected project. The authenticated user is recorded automatically.", tag="Feedback", action_overrides=build_crud_action_overrides( UserFeedbackSerializer, - resource_plural="user feedback entries for the selected tenant", + resource_plural="user feedback entries for the selected project", resource_singular="user feedback entry", ), ) -class UserFeedbackViewSet(TenantOwnedQuerysetMixin, viewsets.ModelViewSet): +class UserFeedbackViewSet(ProjectOwnedQuerysetMixin, viewsets.ModelViewSet): serializer_class = UserFeedbackSerializer - queryset = UserFeedback.objects.select_related("content", "tenant", "user") + queryset = UserFeedback.objects.select_related("content", "project", "user") def perform_create(self, serializer): - serializer.save(tenant=self.get_tenant(), user=self.request.user) + serializer.save(project=self.get_project(), user=self.request.user) -@document_tenant_owned_viewset( +@document_project_owned_viewset( resource_plural="ingestion runs", resource_singular="ingestion run", - create_description="Create a new ingestion run record for the selected tenant to track a content ingestion attempt and its status.", + create_description="Create a new ingestion run record for the selected project to track a content ingestion attempt and its status.", tag="Ingestion", action_overrides=build_crud_action_overrides( IngestionRunSerializer, - resource_plural="ingestion runs for the selected tenant", + resource_plural="ingestion runs for the selected project", resource_singular="ingestion run", ), ) -class IngestionRunViewSet(TenantOwnedQuerysetMixin, viewsets.ModelViewSet): +class IngestionRunViewSet(ProjectOwnedQuerysetMixin, viewsets.ModelViewSet): serializer_class = IngestionRunSerializer - queryset = IngestionRun.objects.select_related("tenant") + queryset = IngestionRun.objects.select_related("project") -@document_tenant_owned_viewset( +@document_project_owned_viewset( resource_plural="source configurations", resource_singular="source configuration", - create_description="Create a new source configuration for the selected tenant. Plugin-specific configuration is validated before the record is saved.", + create_description="Create a new source configuration for the selected project. Plugin-specific configuration is validated before the record is saved.", tag="Ingestion", action_overrides=build_crud_action_overrides( SourceConfigSerializer, - resource_plural="source configurations for the selected tenant", + resource_plural="source configurations for the selected project", resource_singular="source configuration", create_examples=[ SOURCE_CONFIG_CREATE_REQUEST_EXAMPLE, @@ -632,22 +629,22 @@ class IngestionRunViewSet(TenantOwnedQuerysetMixin, viewsets.ModelViewSet): retrieve_examples=[SOURCE_CONFIG_RESPONSE_EXAMPLE], ), ) -class SourceConfigViewSet(TenantOwnedQuerysetMixin, viewsets.ModelViewSet): +class SourceConfigViewSet(ProjectOwnedQuerysetMixin, viewsets.ModelViewSet): serializer_class = SourceConfigSerializer - queryset = SourceConfig.objects.select_related("tenant") + queryset = SourceConfig.objects.select_related("project") -@document_tenant_owned_viewset( +@document_project_owned_viewset( resource_plural="review queue entries", resource_singular="review queue entry", - create_description="Create a new review queue entry for the selected tenant. The referenced content must belong to the same tenant.", + create_description="Create a new review queue entry for the selected project. The referenced content must belong to the same project.", tag="Review Queue", action_overrides=build_crud_action_overrides( ReviewQueueSerializer, - resource_plural="review queue entries for the selected tenant", + resource_plural="review queue entries for the selected project", resource_singular="review queue entry", ), ) -class ReviewQueueViewSet(TenantOwnedQuerysetMixin, viewsets.ModelViewSet): +class ReviewQueueViewSet(ProjectOwnedQuerysetMixin, viewsets.ModelViewSet): serializer_class = ReviewQueueSerializer - queryset = ReviewQueue.objects.select_related("content", "tenant") + queryset = ReviewQueue.objects.select_related("content", "project") diff --git a/core/api_urls.py b/core/api_urls.py index df7532c1..d7cc1cfa 100644 --- a/core/api_urls.py +++ b/core/api_urls.py @@ -5,30 +5,30 @@ ContentViewSet, EntityViewSet, IngestionRunViewSet, + ProjectConfigViewSet, + ProjectViewSet, ReviewQueueViewSet, SkillResultViewSet, SourceConfigViewSet, - TenantConfigViewSet, - TenantViewSet, UserFeedbackViewSet, ) app_name = "api" router = DefaultRouter() -router.register("tenants", TenantViewSet, basename="tenant") +router.register("projects", ProjectViewSet, basename="project") -tenant_router = NestedSimpleRouter(router, r"tenants", lookup="tenant") -tenant_router.register(r"tenant-configs", TenantConfigViewSet, basename="tenant-config") -tenant_router.register(r"entities", EntityViewSet, basename="tenant-entity") -tenant_router.register(r"contents", ContentViewSet, basename="tenant-content") -tenant_router.register(r"skill-results", SkillResultViewSet, basename="tenant-skill-result") -tenant_router.register(r"feedback", UserFeedbackViewSet, basename="tenant-feedback") -tenant_router.register(r"ingestion-runs", IngestionRunViewSet, basename="tenant-ingestion-run") -tenant_router.register(r"source-configs", SourceConfigViewSet, basename="tenant-source-config") -tenant_router.register(r"review-queue", ReviewQueueViewSet, basename="tenant-review-queue") +project_router = NestedSimpleRouter(router, r"projects", lookup="project") +project_router.register(r"project-configs", ProjectConfigViewSet, basename="project-config") +project_router.register(r"entities", EntityViewSet, basename="project-entity") +project_router.register(r"contents", ContentViewSet, basename="project-content") +project_router.register(r"skill-results", SkillResultViewSet, basename="project-skill-result") +project_router.register(r"feedback", UserFeedbackViewSet, basename="project-feedback") +project_router.register(r"ingestion-runs", IngestionRunViewSet, basename="project-ingestion-run") +project_router.register(r"source-configs", SourceConfigViewSet, basename="project-source-config") +project_router.register(r"review-queue", ReviewQueueViewSet, basename="project-review-queue") urlpatterns = [ *router.urls, - *tenant_router.urls, + *project_router.urls, ] diff --git a/core/embeddings.py b/core/embeddings.py index 30dd1dc1..079c2b32 100644 --- a/core/embeddings.py +++ b/core/embeddings.py @@ -91,8 +91,8 @@ def embed_text(self, text: str) -> list[float]: return response.json()["data"][0]["embedding"] -def collection_name_for_tenant(tenant_id: int) -> str: - return f"tenant_{tenant_id}_content" +def collection_name_for_project(project_id: int) -> str: + return f"project_{project_id}_content" @lru_cache(maxsize=1) @@ -122,18 +122,18 @@ def embed_text(text: str) -> list[float]: def upsert_content_embedding(content: Content) -> str: client = get_qdrant_client() - ensure_tenant_collection(content.tenant_id) + ensure_project_collection(content.project_id) embedding_id = content.embedding_id or str(uuid4()) vector = embed_text(build_content_embedding_text(content)) client.upsert( - collection_name=collection_name_for_tenant(content.tenant_id), + collection_name=collection_name_for_project(content.project_id), points=[ PointStruct( id=embedding_id, vector=vector, payload={ "content_id": content.id, - "tenant_id": content.tenant_id, + "project_id": content.project_id, "url": content.url, "title": content.title, "published_date": serialize_published_date(content.published_date), @@ -151,19 +151,19 @@ def upsert_content_embedding(content: Content) -> str: def search_similar( - tenant_id: int, + project_id: int, query_vector: list[float], limit: int = 10, *, is_reference: bool | None = None, exclude_content_id: int | None = None, ): - if not tenant_collection_exists(tenant_id): + if not project_collection_exists(project_id): return [] query_filter = build_search_filter(is_reference=is_reference, exclude_content_id=exclude_content_id) client = cast(Any, get_qdrant_client()) return client.search( - collection_name=collection_name_for_tenant(tenant_id), + collection_name=collection_name_for_project(project_id), query_vector=query_vector, limit=limit, query_filter=query_filter, @@ -173,7 +173,7 @@ def search_similar( def search_similar_content(content: Content, limit: int = 10, *, is_reference: bool | None = None): return search_similar( - content.tenant_id, + content.project_id, embed_text(build_content_embedding_text(content)), limit=limit, is_reference=is_reference, @@ -181,17 +181,17 @@ def search_similar_content(content: Content, limit: int = 10, *, is_reference: b ) -def get_reference_similarity(tenant_id: int, vector: list[float], limit: int = 5) -> float: - scored_points = search_similar(tenant_id, vector, limit=limit, is_reference=True) +def get_reference_similarity(project_id: int, vector: list[float], limit: int = 5) -> float: + scored_points = search_similar(project_id, vector, limit=limit, is_reference=True) if not scored_points: return 0.0 return sum(point.score for point in scored_points) / len(scored_points) -def ensure_tenant_collection(tenant_id: int) -> None: +def ensure_project_collection(project_id: int) -> None: client = get_qdrant_client() - collection_name = collection_name_for_tenant(tenant_id) - if tenant_collection_exists(tenant_id): + collection_name = collection_name_for_project(project_id) + if project_collection_exists(project_id): return client.create_collection( collection_name=collection_name, @@ -199,9 +199,9 @@ def ensure_tenant_collection(tenant_id: int) -> None: ) -def tenant_collection_exists(tenant_id: int) -> bool: +def project_collection_exists(project_id: int) -> bool: try: - get_qdrant_client().get_collection(collection_name_for_tenant(tenant_id)) + get_qdrant_client().get_collection(collection_name_for_project(project_id)) except Exception: return False return True diff --git a/core/management/commands/seed_demo.py b/core/management/commands/seed_demo.py index e848e8d0..d3d058aa 100644 --- a/core/management/commands/seed_demo.py +++ b/core/management/commands/seed_demo.py @@ -5,6 +5,7 @@ from django.conf import settings from django.contrib.auth import get_user_model +from django.contrib.auth.models import Group from django.core.management.base import BaseCommand from django.db import transaction from django.utils import timezone @@ -18,6 +19,8 @@ EntityType, FeedbackType, IngestionRun, + Project, + ProjectConfig, ReviewQueue, ReviewReason, ReviewResolution, @@ -26,8 +29,6 @@ SkillStatus, SourceConfig, SourcePluginName, - Tenant, - TenantConfig, UserFeedback, ) from core.pipeline import ( @@ -36,7 +37,8 @@ SUMMARIZATION_SKILL_NAME, ) -DEMO_TENANT_NAME = "Platform Engineering Weekly" +DEMO_PROJECT_NAME = "Platform Engineering Weekly" +DEMO_GROUP_NAME = "platform-engineering-editors" DEMO_TOPIC_DESCRIPTION = ( "Platform engineering, DevOps, cloud infrastructure, reliability, and " "developer experience." @@ -478,40 +480,40 @@ class Command(BaseCommand): - help = "Seed a deterministic demo tenant with entities, content, pipeline outputs, feedback, and ingestion history." + help = "Seed a deterministic demo project with entities, content, pipeline outputs, feedback, and ingestion history." def handle(self, *args, **options): reference_articles = self._build_reference_articles() sample_articles = self._build_demo_content() with transaction.atomic(): - tenant = self._ensure_demo_tenant() - self._reset_demo_runtime_state(tenant) - entity_map = self._seed_entities(tenant) - source_config_count = self._seed_source_configs(tenant) + project = self._ensure_demo_project() + self._reset_demo_runtime_state(project) + entity_map = self._seed_entities(project) + source_config_count = self._seed_source_configs(project) reference_contents = self._seed_articles( - tenant, + project, reference_articles, entity_map, is_reference=True, source_plugin=REFERENCE_SOURCE_PLUGIN, ) sample_contents = self._seed_articles( - tenant, + project, sample_articles, entity_map, is_reference=False, ) skill_result_count, review_count = self._seed_pipeline_state( - tenant, + project, sample_articles, sample_contents, ) - feedback_count = self._seed_feedback(tenant, sample_contents) - ingestion_run_count = self._seed_ingestion_runs(tenant) + feedback_count = self._seed_feedback(project, sample_contents) + ingestion_run_count = self._seed_ingestion_runs(project) embedded_count = self._sync_embeddings(reference_contents + sample_contents) - self.stdout.write(self.style.SUCCESS(f"Seeded demo tenant: {tenant.name}")) + self.stdout.write(self.style.SUCCESS(f"Seeded demo project: {project.name}")) self.stdout.write(f"Entities: {len(entity_map)}") self.stdout.write(f"Source configs: {source_config_count}") self.stdout.write(f"Reference corpus items: {len(reference_contents)}") @@ -526,7 +528,7 @@ def handle(self, *args, **options): ) ) - def _ensure_demo_tenant(self) -> Tenant: + def _ensure_demo_project(self) -> Project: user_model = get_user_model() user, _ = user_model.objects.get_or_create( username="demo_editor", @@ -534,26 +536,28 @@ def _ensure_demo_tenant(self) -> Tenant: ) user.set_password("demo-password") user.save(update_fields=["password"]) + group, _ = Group.objects.get_or_create(name=DEMO_GROUP_NAME) + user.groups.add(group) - tenant, created = Tenant.objects.get_or_create( - user=user, - name=DEMO_TENANT_NAME, + project, created = Project.objects.get_or_create( + group=group, + name=DEMO_PROJECT_NAME, defaults={"topic_description": DEMO_TOPIC_DESCRIPTION}, ) - if not created and tenant.topic_description != DEMO_TOPIC_DESCRIPTION: - tenant.topic_description = DEMO_TOPIC_DESCRIPTION - tenant.save(update_fields=["topic_description"]) - TenantConfig.objects.get_or_create(tenant=tenant) - return tenant + if not created and project.topic_description != DEMO_TOPIC_DESCRIPTION: + project.topic_description = DEMO_TOPIC_DESCRIPTION + project.save(update_fields=["topic_description"]) + ProjectConfig.objects.get_or_create(project=project) + return project - def _reset_demo_runtime_state(self, tenant: Tenant) -> None: - SkillResult.objects.filter(tenant=tenant).delete() - ReviewQueue.objects.filter(tenant=tenant).delete() - UserFeedback.objects.filter(tenant=tenant).delete() - IngestionRun.objects.filter(tenant=tenant).delete() - SourceConfig.objects.filter(tenant=tenant).delete() + def _reset_demo_runtime_state(self, project: Project) -> None: + SkillResult.objects.filter(project=project).delete() + ReviewQueue.objects.filter(project=project).delete() + UserFeedback.objects.filter(project=project).delete() + IngestionRun.objects.filter(project=project).delete() + SourceConfig.objects.filter(project=project).delete() - def _seed_entities(self, tenant: Tenant) -> dict[str, Entity]: + def _seed_entities(self, project: Project) -> dict[str, Entity]: entities_by_name: dict[str, Entity] = {} for spec in ENTITY_SPECS: defaults = { @@ -568,14 +572,14 @@ def _seed_entities(self, tenant: Tenant) -> dict[str, Entity]: "twitter_handle": spec.get("twitter_handle", ""), } entity, _ = Entity.objects.update_or_create( - tenant=tenant, + project=project, name=spec["name"], defaults=defaults, ) entities_by_name[entity.name] = entity return entities_by_name - def _seed_source_configs(self, tenant: Tenant) -> int: + def _seed_source_configs(self, project: Project) -> int: now = timezone.now() for spec in SOURCE_CONFIG_SPECS: hours_ago = cast(int | None, spec["hours_ago"]) @@ -583,7 +587,7 @@ def _seed_source_configs(self, tenant: Tenant) -> int: if hours_ago is not None: last_fetched_at = now - timedelta(hours=hours_ago) SourceConfig.objects.create( - tenant=tenant, + project=project, plugin_name=cast(str, spec["plugin_name"]), config=spec["config"], is_active=cast(bool, spec["is_active"]), @@ -593,7 +597,7 @@ def _seed_source_configs(self, tenant: Tenant) -> int: def _seed_articles( self, - tenant: Tenant, + project: Project, articles: list[dict[str, Any]], entities_by_name: dict[str, Entity], *, @@ -614,7 +618,7 @@ def _seed_articles( "is_active": True, } content, _ = Content.objects.update_or_create( - tenant=tenant, + project=project, url=article["url"], defaults=defaults, ) @@ -623,7 +627,7 @@ def _seed_articles( def _seed_pipeline_state( self, - tenant: Tenant, + project: Project, article_specs: list[dict[str, Any]], contents: list[Content], ) -> tuple[int, int]: @@ -648,7 +652,7 @@ def _seed_pipeline_state( skill_results.append( SkillResult( content=content, - tenant=tenant, + project=project, skill_name=CLASSIFICATION_SKILL_NAME, status=SkillStatus.COMPLETED, result_data={ @@ -664,7 +668,7 @@ def _seed_pipeline_state( skill_results.append( SkillResult( content=content, - tenant=tenant, + project=project, skill_name=RELEVANCE_SKILL_NAME, status=SkillStatus.COMPLETED, result_data={ @@ -685,7 +689,7 @@ def _seed_pipeline_state( skill_results.append( SkillResult( content=content, - tenant=tenant, + project=project, skill_name=SUMMARIZATION_SKILL_NAME, status=SkillStatus.COMPLETED, result_data={ @@ -711,7 +715,7 @@ def _seed_pipeline_state( ) review_items.append( ReviewQueue( - tenant=tenant, + project=project, content=content, reason=review_reason, confidence=confidence, @@ -728,7 +732,7 @@ def _seed_pipeline_state( ReviewQueue.objects.bulk_create(review_items) return len(skill_results), len(review_items) - def _seed_feedback(self, tenant: Tenant, contents: list[Content]) -> int: + def _seed_feedback(self, project: Project, contents: list[Content]) -> int: user_model = get_user_model() voters = [] for index in range(1, 7): @@ -752,7 +756,7 @@ def _seed_feedback(self, tenant: Tenant, contents: list[Content]) -> int: content=content, user=voters[index % len(voters)], defaults={ - "tenant": tenant, + "project": project, "feedback_type": FeedbackType.UPVOTE, }, ) @@ -763,7 +767,7 @@ def _seed_feedback(self, tenant: Tenant, contents: list[Content]) -> int: content=content, user=voters[(index + 2) % len(voters)], defaults={ - "tenant": tenant, + "project": project, "feedback_type": FeedbackType.DOWNVOTE, }, ) @@ -771,7 +775,7 @@ def _seed_feedback(self, tenant: Tenant, contents: list[Content]) -> int: return feedback_count - def _seed_ingestion_runs(self, tenant: Tenant) -> int: + def _seed_ingestion_runs(self, project: Project) -> int: run_specs = [ { "plugin_name": SourcePluginName.RSS, @@ -833,7 +837,7 @@ def _seed_ingestion_runs(self, tenant: Tenant) -> int: started_hours_ago = cast(int, spec["started_hours_ago"]) duration_minutes = cast(int, spec["duration_minutes"]) run = IngestionRun.objects.create( - tenant=tenant, + project=project, plugin_name=cast(str, spec["plugin_name"]), status=cast(str, spec["status"]), items_fetched=cast(int, spec["items_fetched"]), @@ -1015,7 +1019,7 @@ def _reddit_body(subreddit: str, body: str, band: str, index: int) -> str: "decide whether it is specific enough for platform readers." ) return ( - f"A thread in r/{subreddit} is only loosely connected to the tenant topic. " + f"A thread in r/{subreddit} is only loosely connected to the project topic. " f"{body} The conversation is interesting, but it is more peripheral than the " "other seeded stories." ) diff --git a/core/management/commands/sync_embeddings.py b/core/management/commands/sync_embeddings.py index 6ef61dd9..8cb1836b 100644 --- a/core/management/commands/sync_embeddings.py +++ b/core/management/commands/sync_embeddings.py @@ -8,7 +8,7 @@ class Command(BaseCommand): help = "Backfill Qdrant embeddings for content records." def add_arguments(self, parser): - parser.add_argument("--tenant-id", type=int, help="Only sync content for one tenant.") + parser.add_argument("--project-id", type=int, help="Only sync content for one project.") parser.add_argument("--content-id", type=int, help="Only sync one content record.") parser.add_argument( "--references-only", @@ -17,9 +17,9 @@ def add_arguments(self, parser): ) def handle(self, *args, **options): - queryset = Content.objects.select_related("tenant") - if options["tenant_id"] is not None: - queryset = queryset.filter(tenant_id=options["tenant_id"]) + queryset = Content.objects.select_related("project") + if options["project_id"] is not None: + queryset = queryset.filter(project_id=options["project_id"]) if options["content_id"] is not None: queryset = queryset.filter(pk=options["content_id"]) if options["references_only"]: diff --git a/core/migrations/0001_initial.py b/core/migrations/0001_initial.py index f78712ba..0a01ebc6 100644 --- a/core/migrations/0001_initial.py +++ b/core/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 6.0.4 on 2026-04-20 21:31 +# Generated by Django 6.0.4 on 2026-04-27 16:15 import django.db.models.deletion from django.conf import settings @@ -10,10 +10,40 @@ class Migration(migrations.Migration): initial = True dependencies = [ + ("auth", "0012_alter_user_first_name_max_length"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ + migrations.CreateModel( + name="Project", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=255)), + ("topic_description", models.TextField()), + ("content_retention_days", models.PositiveIntegerField(default=365)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ( + "group", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="projects", + to="auth.group", + ), + ), + ], + options={ + "ordering": ["name"], + }, + ), migrations.CreateModel( name="Entity", fields=[ @@ -47,6 +77,14 @@ class Migration(migrations.Migration): ("mastodon_handle", models.CharField(blank=True, max_length=255)), ("twitter_handle", models.CharField(blank=True, max_length=255)), ("created_at", models.DateTimeField(auto_now_add=True)), + ( + "project", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="entities", + to="core.project", + ), + ), ], options={ "ordering": ["name"], @@ -73,6 +111,8 @@ class Migration(migrations.Migration): ("ingested_at", models.DateTimeField(auto_now_add=True)), ("content_text", models.TextField()), ("relevance_score", models.FloatField(blank=True, null=True)), + ("embedding_id", models.CharField(blank=True, max_length=64)), + ("is_reference", models.BooleanField(default=False)), ("is_active", models.BooleanField(default=True)), ( "entity", @@ -84,13 +124,21 @@ class Migration(migrations.Migration): to="core.entity", ), ), + ( + "project", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="contents", + to="core.project", + ), + ), ], options={ "ordering": ["-published_date"], }, ), migrations.CreateModel( - name="Tenant", + name="ProjectConfig", fields=[ ( "id", @@ -101,21 +149,21 @@ class Migration(migrations.Migration): verbose_name="ID", ), ), - ("name", models.CharField(max_length=255)), - ("topic_description", models.TextField()), - ("content_retention_days", models.PositiveIntegerField(default=365)), - ("created_at", models.DateTimeField(auto_now_add=True)), + ("upvote_authority_weight", models.FloatField(default=0.1)), + ("downvote_authority_weight", models.FloatField(default=-0.05)), + ("authority_decay_rate", models.FloatField(default=0.95)), ( - "user", - models.ForeignKey( + "project", + models.OneToOneField( on_delete=django.db.models.deletion.CASCADE, - related_name="tenants", - to=settings.AUTH_USER_MODEL, + related_name="config", + to="core.project", ), ), ], options={ - "ordering": ["name"], + "verbose_name": "Project config", + "verbose_name_plural": "Project configs", }, ), migrations.CreateModel( @@ -166,11 +214,11 @@ class Migration(migrations.Migration): ), ), ( - "tenant", + "project", models.ForeignKey( on_delete=django.db.models.deletion.CASCADE, related_name="review_queue_items", - to="core.tenant", + to="core.project", ), ), ], @@ -178,26 +226,8 @@ class Migration(migrations.Migration): "ordering": ["resolved", "-created_at"], }, ), - migrations.AddField( - model_name="entity", - name="tenant", - field=models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="entities", - to="core.tenant", - ), - ), - migrations.AddField( - model_name="content", - name="tenant", - field=models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="contents", - to="core.tenant", - ), - ), migrations.CreateModel( - name="TenantConfig", + name="SkillResult", fields=[ ( "id", @@ -208,25 +238,58 @@ class Migration(migrations.Migration): verbose_name="ID", ), ), - ("upvote_authority_weight", models.FloatField(default=0.1)), - ("downvote_authority_weight", models.FloatField(default=-0.05)), - ("authority_decay_rate", models.FloatField(default=0.95)), + ("skill_name", models.CharField(max_length=64)), ( - "tenant", - models.OneToOneField( + "status", + models.CharField( + choices=[ + ("pending", "Pending"), + ("running", "Running"), + ("completed", "Completed"), + ("failed", "Failed"), + ], + max_length=16, + ), + ), + ("result_data", models.JSONField(blank=True, null=True)), + ("error_message", models.TextField(blank=True)), + ("model_used", models.CharField(blank=True, max_length=64)), + ("latency_ms", models.IntegerField(blank=True, null=True)), + ("confidence", models.FloatField(blank=True, null=True)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ( + "content", + models.ForeignKey( on_delete=django.db.models.deletion.CASCADE, - related_name="config", - to="core.tenant", + related_name="skill_results", + to="core.content", + ), + ), + ( + "project", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="skill_results", + to="core.project", + ), + ), + ( + "superseded_by", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="supersedes", + to="core.skillresult", ), ), ], options={ - "verbose_name": "Tenant config", - "verbose_name_plural": "Tenant configs", + "ordering": ["-created_at"], }, ), migrations.CreateModel( - name="UserFeedback", + name="SourceConfig", fields=[ ( "id", @@ -238,44 +301,29 @@ class Migration(migrations.Migration): ), ), ( - "feedback_type", + "plugin_name", models.CharField( - choices=[("upvote", "Upvote"), ("downvote", "Downvote")], - max_length=16, - ), - ), - ("created_at", models.DateTimeField(auto_now_add=True)), - ( - "content", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="feedback", - to="core.content", - ), - ), - ( - "tenant", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="feedback", - to="core.tenant", + choices=[("rss", "RSS"), ("reddit", "Reddit")], max_length=64 ), ), + ("config", models.JSONField(default=dict)), + ("is_active", models.BooleanField(default=True)), + ("last_fetched_at", models.DateTimeField(blank=True, null=True)), ( - "user", + "project", models.ForeignKey( on_delete=django.db.models.deletion.CASCADE, - related_name="content_feedback", - to=settings.AUTH_USER_MODEL, + related_name="source_configs", + to="core.project", ), ), ], options={ - "ordering": ["-created_at"], + "ordering": ["plugin_name", "id"], }, ), migrations.CreateModel( - name="SkillResult", + name="UserFeedback", fields=[ ( "id", @@ -286,64 +334,41 @@ class Migration(migrations.Migration): verbose_name="ID", ), ), - ("skill_name", models.CharField(max_length=64)), ( - "status", + "feedback_type", models.CharField( - choices=[ - ("pending", "Pending"), - ("running", "Running"), - ("completed", "Completed"), - ("failed", "Failed"), - ], + choices=[("upvote", "Upvote"), ("downvote", "Downvote")], max_length=16, ), ), - ("result_data", models.JSONField(blank=True, null=True)), - ("error_message", models.TextField(blank=True)), - ("model_used", models.CharField(blank=True, max_length=64)), - ("latency_ms", models.IntegerField(blank=True, null=True)), - ("confidence", models.FloatField(blank=True, null=True)), ("created_at", models.DateTimeField(auto_now_add=True)), ( "content", models.ForeignKey( on_delete=django.db.models.deletion.CASCADE, - related_name="skill_results", + related_name="feedback", to="core.content", ), ), ( - "superseded_by", + "project", models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name="supersedes", - to="core.skillresult", + on_delete=django.db.models.deletion.CASCADE, + related_name="feedback", + to="core.project", ), ), ( - "tenant", + "user", models.ForeignKey( on_delete=django.db.models.deletion.CASCADE, - related_name="skill_results", - to="core.tenant", + related_name="content_feedback", + to=settings.AUTH_USER_MODEL, ), ), ], options={ "ordering": ["-created_at"], - "indexes": [ - models.Index( - fields=["content", "skill_name"], - name="core_skillr_content_0d49f9_idx", - ), - models.Index( - fields=["tenant", "created_at"], - name="core_skillr_tenant__1bde27_idx", - ), - ], }, ), migrations.CreateModel( @@ -376,11 +401,11 @@ class Migration(migrations.Migration): ("items_ingested", models.IntegerField(default=0)), ("error_message", models.TextField(blank=True)), ( - "tenant", + "project", models.ForeignKey( on_delete=django.db.models.deletion.CASCADE, related_name="ingestion_runs", - to="core.tenant", + to="core.project", ), ), ], @@ -388,8 +413,8 @@ class Migration(migrations.Migration): "ordering": ["-started_at"], "indexes": [ models.Index( - fields=["tenant", "plugin_name", "-started_at"], - name="core_ingest_tenant__874035_idx", + fields=["project", "plugin_name", "-started_at"], + name="core_ingest_project_fd3a74_idx", ) ], }, @@ -397,27 +422,53 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="entity", constraint=models.UniqueConstraint( - fields=("tenant", "name"), name="core_entity_unique_tenant_name" + fields=("project", "name"), name="core_entity_unique_project_name" ), ), migrations.AddIndex( model_name="content", index=models.Index( - fields=["tenant", "-published_date"], - name="core_conten_tenant__01494c_idx", + fields=["project", "-published_date"], + name="core_conten_project_6662d0_idx", ), ), migrations.AddIndex( model_name="content", index=models.Index( - fields=["tenant", "-relevance_score"], - name="core_conten_tenant__a1c995_idx", + fields=["project", "-relevance_score"], + name="core_conten_project_127912_idx", + ), + ), + migrations.AddIndex( + model_name="content", + index=models.Index( + fields=["project", "is_reference"], + name="core_conten_project_c689be_idx", ), ), migrations.AddIndex( model_name="content", index=models.Index(fields=["url"], name="core_conten_url_4d8416_idx"), ), + migrations.AddIndex( + model_name="skillresult", + index=models.Index( + fields=["content", "skill_name"], name="core_skillr_content_0d49f9_idx" + ), + ), + migrations.AddIndex( + model_name="skillresult", + index=models.Index( + fields=["project", "created_at"], name="core_skillr_project_60360b_idx" + ), + ), + migrations.AddIndex( + model_name="sourceconfig", + index=models.Index( + fields=["project", "plugin_name", "is_active"], + name="core_source_project_f1abc6_idx", + ), + ), migrations.AddConstraint( model_name="userfeedback", constraint=models.UniqueConstraint( diff --git a/core/migrations/0002_sourceconfig.py b/core/migrations/0002_sourceconfig.py deleted file mode 100644 index 30f1bf9b..00000000 --- a/core/migrations/0002_sourceconfig.py +++ /dev/null @@ -1,47 +0,0 @@ -# Generated by Django 6.0.4 on 2026-04-21 00:00 - -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("core", "0001_initial"), - ] - - operations = [ - migrations.CreateModel( - name="SourceConfig", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("plugin_name", models.CharField(choices=[("rss", "RSS"), ("reddit", "Reddit")], max_length=64)), - ("config", models.JSONField(default=dict)), - ("is_active", models.BooleanField(default=True)), - ("last_fetched_at", models.DateTimeField(blank=True, null=True)), - ( - "tenant", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="source_configs", - to="core.tenant", - ), - ), - ], - options={ - "ordering": ["plugin_name", "id"], - }, - ), - migrations.AddIndex( - model_name="sourceconfig", - index=models.Index(fields=["tenant", "plugin_name", "is_active"], name="core_source_tenant__b8437a_idx"), - ), - ] diff --git a/core/migrations/0003_content_embeddings.py b/core/migrations/0003_content_embeddings.py deleted file mode 100644 index 24cffcba..00000000 --- a/core/migrations/0003_content_embeddings.py +++ /dev/null @@ -1,27 +0,0 @@ -# Generated by Django 6.0.4 on 2026-04-21 00:00 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("core", "0002_sourceconfig"), - ] - - operations = [ - migrations.AddField( - model_name="content", - name="embedding_id", - field=models.CharField(blank=True, max_length=64), - ), - migrations.AddField( - model_name="content", - name="is_reference", - field=models.BooleanField(default=False), - ), - migrations.AddIndex( - model_name="content", - index=models.Index(fields=["tenant", "is_reference"], name="core_conten_tenant__e80c35_idx"), - ), - ] diff --git a/core/models.py b/core/models.py index 31cf5852..e23f7dfe 100644 --- a/core/models.py +++ b/core/models.py @@ -1,4 +1,5 @@ from django.conf import settings +from django.contrib.auth.models import Group from django.db import models @@ -41,9 +42,9 @@ class ReviewResolution(models.TextChoices): HUMAN_REJECTED = "human_rejected", "Human Rejected" -class Tenant(models.Model): +class Project(models.Model): name = models.CharField(max_length=255) - user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="tenants") + group = models.ForeignKey(Group, on_delete=models.CASCADE, related_name="projects") topic_description = models.TextField() content_retention_days = models.PositiveIntegerField(default=365) created_at = models.DateTimeField(auto_now_add=True) @@ -55,22 +56,22 @@ def __str__(self) -> str: return self.name -class TenantConfig(models.Model): - tenant = models.OneToOneField(Tenant, on_delete=models.CASCADE, related_name="config") +class ProjectConfig(models.Model): + project = models.OneToOneField(Project, on_delete=models.CASCADE, related_name="config") upvote_authority_weight = models.FloatField(default=0.1) downvote_authority_weight = models.FloatField(default=-0.05) authority_decay_rate = models.FloatField(default=0.95) class Meta: - verbose_name = "Tenant config" - verbose_name_plural = "Tenant configs" + verbose_name = "Project config" + verbose_name_plural = "Project configs" def __str__(self) -> str: - return f"Config for {self.tenant.name}" + return f"Config for {self.project.name}" class Entity(models.Model): - tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE, related_name="entities") + project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name="entities") name = models.CharField(max_length=255) type = models.CharField(max_length=32, choices=EntityType.choices) description = models.TextField(blank=True) @@ -86,7 +87,7 @@ class Entity(models.Model): class Meta: ordering = ["name"] constraints = [ - models.UniqueConstraint(fields=["tenant", "name"], name="core_entity_unique_tenant_name"), + models.UniqueConstraint(fields=["project", "name"], name="core_entity_unique_project_name"), ] def __str__(self) -> str: @@ -94,7 +95,7 @@ def __str__(self) -> str: class Content(models.Model): - tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE, related_name="contents") + project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name="contents") url = models.URLField() title = models.CharField(max_length=512) author = models.CharField(max_length=255, blank=True) @@ -112,9 +113,9 @@ class Content(models.Model): class Meta: ordering = ["-published_date"] indexes = [ - models.Index(fields=["tenant", "-published_date"]), - models.Index(fields=["tenant", "-relevance_score"]), - models.Index(fields=["tenant", "is_reference"]), + models.Index(fields=["project", "-published_date"]), + models.Index(fields=["project", "-relevance_score"]), + models.Index(fields=["project", "is_reference"]), models.Index(fields=["url"]), ] @@ -124,7 +125,7 @@ def __str__(self) -> str: class SkillResult(models.Model): content = models.ForeignKey(Content, on_delete=models.CASCADE, related_name="skill_results") - tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE, related_name="skill_results") + project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name="skill_results") skill_name = models.CharField(max_length=64) status = models.CharField(max_length=16, choices=SkillStatus.choices) result_data = models.JSONField(null=True, blank=True) @@ -145,7 +146,7 @@ class Meta: ordering = ["-created_at"] indexes = [ models.Index(fields=["content", "skill_name"]), - models.Index(fields=["tenant", "created_at"]), + models.Index(fields=["project", "created_at"]), ] def __str__(self) -> str: @@ -154,7 +155,7 @@ def __str__(self) -> str: class UserFeedback(models.Model): content = models.ForeignKey(Content, on_delete=models.CASCADE, related_name="feedback") - tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE, related_name="feedback") + project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name="feedback") user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="content_feedback") feedback_type = models.CharField(max_length=16, choices=FeedbackType.choices) created_at = models.DateTimeField(auto_now_add=True) @@ -170,7 +171,7 @@ def __str__(self) -> str: class SourceConfig(models.Model): - tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE, related_name="source_configs") + project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name="source_configs") plugin_name = models.CharField(max_length=64, choices=SourcePluginName.choices) config = models.JSONField(default=dict) is_active = models.BooleanField(default=True) @@ -179,15 +180,15 @@ class SourceConfig(models.Model): class Meta: ordering = ["plugin_name", "id"] indexes = [ - models.Index(fields=["tenant", "plugin_name", "is_active"]), + models.Index(fields=["project", "plugin_name", "is_active"]), ] def __str__(self) -> str: - return f"{self.plugin_name} source for {self.tenant.name}" + return f"{self.plugin_name} source for {self.project.name}" class IngestionRun(models.Model): - tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE, related_name="ingestion_runs") + project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name="ingestion_runs") plugin_name = models.CharField(max_length=64) started_at = models.DateTimeField(auto_now_add=True) completed_at = models.DateTimeField(null=True, blank=True) @@ -199,15 +200,15 @@ class IngestionRun(models.Model): class Meta: ordering = ["-started_at"] indexes = [ - models.Index(fields=["tenant", "plugin_name", "-started_at"]), + models.Index(fields=["project", "plugin_name", "-started_at"]), ] def __str__(self) -> str: - return f"{self.plugin_name} for {self.tenant.name}" + return f"{self.plugin_name} for {self.project.name}" class ReviewQueue(models.Model): - tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE, related_name="review_queue_items") + project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name="review_queue_items") content = models.ForeignKey(Content, on_delete=models.CASCADE, related_name="review_queue_items") reason = models.CharField(max_length=64, choices=ReviewReason.choices) confidence = models.FloatField() diff --git a/core/pipeline.py b/core/pipeline.py index 611b081a..ac5527c4 100644 --- a/core/pipeline.py +++ b/core/pipeline.py @@ -33,7 +33,7 @@ class PipelineState(TypedDict, total=False): content_id: int - tenant_id: int + project_id: int classification: dict[str, Any] | None relevance: dict[str, Any] | None summary: dict[str, Any] | None @@ -66,10 +66,10 @@ def get_ingestion_graph(): def process_content_pipeline(content_id: int) -> PipelineState: - content = Content.objects.select_related("tenant").get(pk=content_id) + content = Content.objects.select_related("project").get(pk=content_id) initial_state: PipelineState = { "content_id": content.id, - "tenant_id": content.tenant_id, + "project_id": content.project_id, "status": "processing", } return get_ingestion_graph().invoke(initial_state) @@ -194,7 +194,7 @@ def run_content_classification(content: Content) -> dict[str, Any]: def run_relevance_scoring(content: Content) -> dict[str, Any]: vector = embed_text(build_content_embedding_text(content)) - similarity = float(get_reference_similarity(content.tenant_id, vector)) + similarity = float(get_reference_similarity(content.project_id, vector)) if similarity >= settings.AI_RELEVANCE_HIGH_THRESHOLD or similarity < settings.AI_RELEVANCE_LOW_THRESHOLD: return { "relevance_score": similarity, @@ -213,7 +213,7 @@ def run_relevance_scoring(content: Content) -> dict[str, Any]: "Return JSON with relevance_score between 0 and 1, explanation, and used_llm=true." ), user_prompt=( - f"Newsletter topic: {content.tenant.topic_description}\n" + f"Newsletter topic: {content.project.topic_description}\n" f"Reference similarity score: {similarity:.3f}\n" f"Title: {content.title}\n" f"Content:\n{content.content_text[:5000]}" @@ -236,8 +236,8 @@ def run_relevance_scoring(content: Content) -> dict[str, Any]: return { "relevance_score": similarity, "explanation": ( - f"Borderline reference similarity of {similarity:.2f} against the tenant baseline for " - f"'{content.tenant.topic_description}'." + f"Borderline reference similarity of {similarity:.2f} against the project baseline for " + f"'{content.project.topic_description}'." ), "used_llm": False, "model_used": f"embedding:{settings.EMBEDDING_MODEL}", @@ -254,7 +254,7 @@ def run_summarization(content: Content) -> dict[str, Any]: "You write concise newsletter-ready summaries. Return JSON with a single key named summary." ), user_prompt=( - f"Newsletter topic: {content.tenant.topic_description}\n" + f"Newsletter topic: {content.project.topic_description}\n" f"Title: {content.title}\n" f"Content:\n{content.content_text[:5000]}" ), @@ -299,7 +299,7 @@ def create_pending_skill_result(content: Content, skill_name: str) -> SkillResul def execute_background_skill_result(skill_result_id: int, skill_name: str) -> SkillResult: - skill_result = SkillResult.objects.select_related("content", "content__tenant").get(pk=skill_result_id) + skill_result = SkillResult.objects.select_related("content", "content__project").get(pk=skill_result_id) if skill_result.skill_name != skill_name: raise ValueError( f"Skill result {skill_result.id} is for {skill_result.skill_name}, not {skill_name}." @@ -523,7 +523,7 @@ def _clamp_score(value: Any) -> float: def _get_content(state: PipelineState) -> Content: - return Content.objects.select_related("tenant").get(pk=state["content_id"]) + return Content.objects.select_related("project").get(pk=state["content_id"]) def _upsert_review_queue_item(content: Content, *, reason: ReviewReason, confidence: float) -> ReviewQueue: @@ -533,7 +533,7 @@ def _upsert_review_queue_item(content: Content, *, reason: ReviewReason, confide existing.save(update_fields=["confidence"]) return existing return ReviewQueue.objects.create( - tenant=content.tenant, + project=content.project, content=content, reason=reason, confidence=confidence, @@ -554,7 +554,7 @@ def _create_skill_result( previous = SkillResult.objects.filter(content=content, skill_name=skill_name, superseded_by__isnull=True).first() skill_result = SkillResult.objects.create( content=content, - tenant=content.tenant, + project=content.project, skill_name=skill_name, status=status, result_data=result_data, diff --git a/core/plugins/base.py b/core/plugins/base.py index 672e91b9..51fa67d2 100644 --- a/core/plugins/base.py +++ b/core/plugins/base.py @@ -21,7 +21,7 @@ class SourcePlugin(ABC): def __init__(self, source_config): self.source_config = source_config - self.tenant = source_config.tenant + self.project = source_config.project @classmethod def validate_config(cls, config: object) -> dict: @@ -45,7 +45,7 @@ def match_entity_for_url(self, url: str): target_hostname = self._normalize_hostname(url) if not target_hostname: return None - for entity in self.tenant.entities.exclude(website_url=""): + for entity in self.project.entities.exclude(website_url=""): if self._normalize_hostname(entity.website_url) == target_hostname: return entity return None diff --git a/core/serializers.py b/core/serializers.py index 5fd20697..7f56ef29 100644 --- a/core/serializers.py +++ b/core/serializers.py @@ -1,34 +1,39 @@ +from django.contrib.auth.models import Group from rest_framework import serializers from core.models import ( Content, Entity, IngestionRun, + Project, + ProjectConfig, ReviewQueue, SkillResult, SourceConfig, - Tenant, - TenantConfig, UserFeedback, ) from core.plugins import validate_plugin_config -class TenantScopedSerializerMixin: +class ProjectScopedSerializerMixin: def _filter_related_queryset(self, request): user = request.user - tenant = self.context.get("tenant") - if "tenant" in self.fields: - self.fields["tenant"].queryset = Tenant.objects.filter(user=user) + project = self.context.get("project") + if "group" in self.fields: + self.fields["group"].queryset = Group.objects.filter(user=user) + if "project" in self.fields: + self.fields["project"].queryset = Project.objects.filter(group__user=user).distinct() if "entity" in self.fields: - entity_queryset = Entity.objects.filter(tenant=tenant) if tenant else Entity.objects.filter(tenant__user=user) + entity_queryset = Entity.objects.filter(project=project) if project else Entity.objects.filter(project__group__user=user) self.fields["entity"].queryset = entity_queryset if "content" in self.fields: - content_queryset = Content.objects.filter(tenant=tenant) if tenant else Content.objects.filter(tenant__user=user) + content_queryset = Content.objects.filter(project=project) if project else Content.objects.filter(project__group__user=user) self.fields["content"].queryset = content_queryset if "superseded_by" in self.fields: skill_result_queryset = ( - SkillResult.objects.filter(tenant=tenant) if tenant else SkillResult.objects.filter(tenant__user=user) + SkillResult.objects.filter(project=project) + if project + else SkillResult.objects.filter(project__group__user=user) ) self.fields["superseded_by"].queryset = skill_result_queryset @@ -39,34 +44,33 @@ def __init__(self, *args, **kwargs): self._filter_related_queryset(request) -class TenantSerializer(serializers.ModelSerializer): - user = serializers.PrimaryKeyRelatedField(read_only=True) +class ProjectSerializer(ProjectScopedSerializerMixin, serializers.ModelSerializer): class Meta: - model = Tenant - fields = ["id", "name", "user", "topic_description", "content_retention_days", "created_at"] - read_only_fields = ["id", "user", "created_at"] + model = Project + fields = ["id", "name", "group", "topic_description", "content_retention_days", "created_at"] + read_only_fields = ["id", "created_at"] -class TenantConfigSerializer(TenantScopedSerializerMixin, serializers.ModelSerializer): +class ProjectConfigSerializer(ProjectScopedSerializerMixin, serializers.ModelSerializer): class Meta: - model = TenantConfig + model = ProjectConfig fields = [ "id", - "tenant", + "project", "upvote_authority_weight", "downvote_authority_weight", "authority_decay_rate", ] - read_only_fields = ["id", "tenant"] + read_only_fields = ["id", "project"] -class EntitySerializer(TenantScopedSerializerMixin, serializers.ModelSerializer): +class EntitySerializer(ProjectScopedSerializerMixin, serializers.ModelSerializer): class Meta: model = Entity fields = [ "id", - "tenant", + "project", "name", "type", "description", @@ -79,15 +83,15 @@ class Meta: "twitter_handle", "created_at", ] - read_only_fields = ["id", "tenant", "created_at"] + read_only_fields = ["id", "project", "created_at"] -class ContentSerializer(TenantScopedSerializerMixin, serializers.ModelSerializer): +class ContentSerializer(ProjectScopedSerializerMixin, serializers.ModelSerializer): class Meta: model = Content fields = [ "id", - "tenant", + "project", "url", "title", "author", @@ -102,23 +106,23 @@ class Meta: "is_reference", "is_active", ] - read_only_fields = ["id", "tenant", "ingested_at", "embedding_id"] + read_only_fields = ["id", "project", "ingested_at", "embedding_id"] def validate(self, attrs): - tenant = self.context.get("tenant") or attrs.get("tenant") or getattr(self.instance, "tenant", None) + project = self.context.get("project") or attrs.get("project") or getattr(self.instance, "project", None) entity = attrs.get("entity") or getattr(self.instance, "entity", None) - if tenant and entity and entity.tenant_id != tenant.id: - raise serializers.ValidationError({"entity": "Entity must belong to the selected tenant."}) + if project and entity and entity.project_id != project.id: + raise serializers.ValidationError({"entity": "Entity must belong to the selected project."}) return attrs -class SkillResultSerializer(TenantScopedSerializerMixin, serializers.ModelSerializer): +class SkillResultSerializer(ProjectScopedSerializerMixin, serializers.ModelSerializer): class Meta: model = SkillResult fields = [ "id", "content", - "tenant", + "project", "skill_name", "status", "result_data", @@ -129,38 +133,38 @@ class Meta: "created_at", "superseded_by", ] - read_only_fields = ["id", "tenant", "created_at"] + read_only_fields = ["id", "project", "created_at"] def validate(self, attrs): - tenant = self.context.get("tenant") or attrs.get("tenant") or getattr(self.instance, "tenant", None) + project = self.context.get("project") or attrs.get("project") or getattr(self.instance, "project", None) content = attrs.get("content") or getattr(self.instance, "content", None) - if tenant and content and content.tenant_id != tenant.id: - raise serializers.ValidationError({"content": "Content must belong to the selected tenant."}) + if project and content and content.project_id != project.id: + raise serializers.ValidationError({"content": "Content must belong to the selected project."}) return attrs -class UserFeedbackSerializer(TenantScopedSerializerMixin, serializers.ModelSerializer): +class UserFeedbackSerializer(ProjectScopedSerializerMixin, serializers.ModelSerializer): user = serializers.PrimaryKeyRelatedField(read_only=True) class Meta: model = UserFeedback - fields = ["id", "content", "tenant", "user", "feedback_type", "created_at"] - read_only_fields = ["id", "tenant", "user", "created_at"] + fields = ["id", "content", "project", "user", "feedback_type", "created_at"] + read_only_fields = ["id", "project", "user", "created_at"] def validate(self, attrs): - tenant = self.context.get("tenant") or attrs.get("tenant") or getattr(self.instance, "tenant", None) + project = self.context.get("project") or attrs.get("project") or getattr(self.instance, "project", None) content = attrs.get("content") or getattr(self.instance, "content", None) - if tenant and content and content.tenant_id != tenant.id: - raise serializers.ValidationError({"content": "Content must belong to the selected tenant."}) + if project and content and content.project_id != project.id: + raise serializers.ValidationError({"content": "Content must belong to the selected project."}) return attrs -class IngestionRunSerializer(TenantScopedSerializerMixin, serializers.ModelSerializer): +class IngestionRunSerializer(ProjectScopedSerializerMixin, serializers.ModelSerializer): class Meta: model = IngestionRun fields = [ "id", - "tenant", + "project", "plugin_name", "started_at", "completed_at", @@ -169,14 +173,14 @@ class Meta: "items_ingested", "error_message", ] - read_only_fields = ["id", "tenant", "started_at"] + read_only_fields = ["id", "project", "started_at"] -class SourceConfigSerializer(TenantScopedSerializerMixin, serializers.ModelSerializer): +class SourceConfigSerializer(ProjectScopedSerializerMixin, serializers.ModelSerializer): class Meta: model = SourceConfig - fields = ["id", "tenant", "plugin_name", "config", "is_active", "last_fetched_at"] - read_only_fields = ["id", "tenant", "last_fetched_at"] + fields = ["id", "project", "plugin_name", "config", "is_active", "last_fetched_at"] + read_only_fields = ["id", "project", "last_fetched_at"] def validate(self, attrs): plugin_name = attrs.get("plugin_name") or getattr(self.instance, "plugin_name", None) @@ -189,15 +193,15 @@ def validate(self, attrs): return attrs -class ReviewQueueSerializer(TenantScopedSerializerMixin, serializers.ModelSerializer): +class ReviewQueueSerializer(ProjectScopedSerializerMixin, serializers.ModelSerializer): class Meta: model = ReviewQueue - fields = ["id", "tenant", "content", "reason", "confidence", "created_at", "resolved", "resolution"] - read_only_fields = ["id", "tenant", "created_at"] + fields = ["id", "project", "content", "reason", "confidence", "created_at", "resolved", "resolution"] + read_only_fields = ["id", "project", "created_at"] def validate(self, attrs): - tenant = self.context.get("tenant") or attrs.get("tenant") or getattr(self.instance, "tenant", None) + project = self.context.get("project") or attrs.get("project") or getattr(self.instance, "project", None) content = attrs.get("content") or getattr(self.instance, "content", None) - if tenant and content and content.tenant_id != tenant.id: - raise serializers.ValidationError({"content": "Content must belong to the selected tenant."}) + if project and content and content.project_id != project.id: + raise serializers.ValidationError({"content": "Content must belong to the selected project."}) return attrs diff --git a/core/tasks.py b/core/tasks.py index 1ed8cb6a..eb1568c3 100644 --- a/core/tasks.py +++ b/core/tasks.py @@ -20,9 +20,9 @@ @shared_task(name="core.tasks.run_ingestion") def run_ingestion(source_config_id: int): - source_config = SourceConfig.objects.select_related("tenant").get(pk=source_config_id) + source_config = SourceConfig.objects.select_related("project").get(pk=source_config_id) ingestion_run = IngestionRun.objects.create( - tenant=source_config.tenant, + project=source_config.project, plugin_name=source_config.plugin_name, status=RunStatus.RUNNING, ) @@ -95,10 +95,10 @@ def _ingest_source_config(source_config: SourceConfig) -> tuple[int, int]: fetched_items = plugin.fetch_new_content(source_config.last_fetched_at) ingested_count = 0 for item in fetched_items: - if Content.objects.filter(tenant=source_config.tenant, url=item.url).exists(): + if Content.objects.filter(project=source_config.project, url=item.url).exists(): continue content = Content.objects.create( - tenant=source_config.tenant, + project=source_config.project, entity=plugin.match_entity_for_url(item.url), url=item.url, title=item.title[:512], diff --git a/core/tests/test_admin.py b/core/tests/test_admin.py index 301cb7a0..d0c3e33d 100644 --- a/core/tests/test_admin.py +++ b/core/tests/test_admin.py @@ -4,6 +4,7 @@ import pytest from django.contrib import messages from django.contrib.admin.sites import AdminSite +from django.contrib.auth.models import Group from django.utils import timezone from core.admin import ( @@ -15,12 +16,12 @@ from core.models import ( Content, IngestionRun, + Project, ReviewQueue, ReviewReason, RunStatus, SourceConfig, SourcePluginName, - Tenant, ) pytestmark = pytest.mark.django_db @@ -29,13 +30,15 @@ @pytest.fixture def source_admin_context(django_user_model): user = django_user_model.objects.create_user(username="admin-owner", password="testpass123") - tenant = Tenant.objects.create(name="Admin Tenant", user=user, topic_description="Infra") - return SimpleNamespace(user=user, tenant=tenant) + group = Group.objects.create(name="admin-team") + user.groups.add(group) + project = Project.objects.create(name="Admin Project", group=group, topic_description="Infra") + return SimpleNamespace(user=user, group=group, project=project) def test_test_source_connection_reports_success(source_admin_context, mocker): source_config = SourceConfig.objects.create( - tenant=source_admin_context.tenant, + project=source_admin_context.project, plugin_name=SourcePluginName.RSS, config={"feed_url": "https://example.com/feed.xml"}, ) @@ -66,7 +69,7 @@ def test_test_source_connection_reports_success(source_admin_context, mocker): def test_test_source_connection_reports_failures(source_admin_context, mocker): source_config = SourceConfig.objects.create( - tenant=source_admin_context.tenant, + project=source_admin_context.project, plugin_name=SourcePluginName.RSS, config={"feed_url": "https://example.com/feed.xml"}, ) @@ -84,14 +87,14 @@ def test_test_source_connection_reports_failures(source_admin_context, mocker): admin_instance.message_user.assert_called_once_with( ANY, - "Connectivity check failed for: rss source for Admin Tenant: Missing required config field: feed_url", + "Connectivity check failed for: rss source for Admin Project: Missing required config field: feed_url", messages.ERROR, ) def test_source_config_display_health_renders_without_django6_format_html_error(source_admin_context): source_config = SourceConfig.objects.create( - tenant=source_admin_context.tenant, + project=source_admin_context.project, plugin_name=SourcePluginName.RSS, config={"feed_url": "https://example.com/feed.xml"}, is_active=True, @@ -106,7 +109,7 @@ def test_source_config_display_health_renders_without_django6_format_html_error( def test_review_queue_changelist_view_builds_dashboard_stats(source_admin_context, mocker): content = Content.objects.create( - tenant=source_admin_context.tenant, + project=source_admin_context.project, url="https://example.com/review-item", title="Review Item", author="Reviewer", @@ -115,7 +118,7 @@ def test_review_queue_changelist_view_builds_dashboard_stats(source_admin_contex content_text="Review queue content", ) ReviewQueue.objects.create( - tenant=source_admin_context.tenant, + project=source_admin_context.project, content=content, reason=ReviewReason.BORDERLINE_RELEVANCE, confidence=0.42, @@ -137,7 +140,7 @@ def test_review_queue_changelist_view_builds_dashboard_stats(source_admin_contex def test_review_queue_display_confidence_renders_without_django6_format_error(source_admin_context): content = Content.objects.create( - tenant=source_admin_context.tenant, + project=source_admin_context.project, url="https://example.com/review-confidence", title="Review Confidence", author="Reviewer", @@ -146,7 +149,7 @@ def test_review_queue_display_confidence_renders_without_django6_format_error(so content_text="Review queue content", ) review_item = ReviewQueue.objects.create( - tenant=source_admin_context.tenant, + project=source_admin_context.project, content=content, reason=ReviewReason.BORDERLINE_RELEVANCE, confidence=0.42, @@ -161,7 +164,7 @@ def test_review_queue_display_confidence_renders_without_django6_format_error(so def test_ingestion_run_display_efficiency_renders_without_django6_format_error(source_admin_context): run = IngestionRun.objects.create( - tenant=source_admin_context.tenant, + project=source_admin_context.project, plugin_name=SourcePluginName.RSS, status=RunStatus.SUCCESS, items_fetched=12, @@ -176,7 +179,7 @@ def test_ingestion_run_display_efficiency_renders_without_django6_format_error(s def test_content_preview_uses_content_text(source_admin_context): content = Content.objects.create( - tenant=source_admin_context.tenant, + project=source_admin_context.project, url="https://example.com/admin-preview", title="Admin Preview", author="Editor", @@ -193,7 +196,7 @@ def test_content_preview_uses_content_text(source_admin_context): def test_content_preview_returns_dash_when_content_text_blank(source_admin_context): content = Content.objects.create( - tenant=source_admin_context.tenant, + project=source_admin_context.project, url="https://example.com/admin-preview-empty", title="Admin Preview Empty", author="Editor", diff --git a/core/tests/test_api.py b/core/tests/test_api.py index 5e8e1349..d9063874 100644 --- a/core/tests/test_api.py +++ b/core/tests/test_api.py @@ -2,6 +2,7 @@ from unittest.mock import patch from django.contrib.auth import get_user_model +from django.contrib.auth.models import Group from django.urls import reverse from rest_framework import status from rest_framework.test import APITestCase @@ -11,6 +12,8 @@ Entity, FeedbackType, IngestionRun, + Project, + ProjectConfig, ReviewQueue, ReviewReason, RunStatus, @@ -18,39 +21,41 @@ SkillStatus, SourceConfig, SourcePluginName, - Tenant, - TenantConfig, UserFeedback, ) -class TenantScopedApiTests(APITestCase): +class ProjectScopedApiTests(APITestCase): def setUp(self): user_model = get_user_model() self.owner = user_model.objects.create_user(username="owner", password="testpass123") self.other_user = user_model.objects.create_user(username="other", password="testpass123") - self.owner_tenant = Tenant.objects.create( - name="Owner Tenant", - user=self.owner, + self.owner_group = Group.objects.create(name="owner-team") + self.owner.groups.add(self.owner_group) + self.other_group = Group.objects.create(name="other-team") + self.other_user.groups.add(self.other_group) + self.owner_project = Project.objects.create( + name="Owner Project", + group=self.owner_group, topic_description="Platform engineering", ) - self.other_tenant = Tenant.objects.create( - name="Other Tenant", - user=self.other_user, + self.other_project = Project.objects.create( + name="Other Project", + group=self.other_group, topic_description="Frontend", ) self.owner_entity = Entity.objects.create( - tenant=self.owner_tenant, + project=self.owner_project, name="Owner Entity", type="individual", ) self.other_entity = Entity.objects.create( - tenant=self.other_tenant, + project=self.other_project, name="Other Entity", type="vendor", ) self.owner_content = Content.objects.create( - tenant=self.owner_tenant, + project=self.owner_project, url="https://example.com/owner", title="Owner Content", author="Owner Author", @@ -60,7 +65,7 @@ def setUp(self): content_text="Owner content text", ) self.other_content = Content.objects.create( - tenant=self.other_tenant, + project=self.other_project, url="https://example.com/other", title="Other Content", author="Other Author", @@ -69,29 +74,29 @@ def setUp(self): published_date="2026-04-21T00:00:00Z", content_text="Other content text", ) - self.owner_config = TenantConfig.objects.create(tenant=self.owner_tenant) + self.owner_config = ProjectConfig.objects.create(project=self.owner_project) self.owner_skill_result = SkillResult.objects.create( - tenant=self.owner_tenant, + project=self.owner_project, content=self.owner_content, skill_name="summarization", status=SkillStatus.COMPLETED, result_data={"summary": "Owner summary"}, ) self.owner_ingestion_run = IngestionRun.objects.create( - tenant=self.owner_tenant, + project=self.owner_project, plugin_name="rss", status=RunStatus.SUCCESS, items_fetched=5, items_ingested=4, ) self.owner_review_queue = ReviewQueue.objects.create( - tenant=self.owner_tenant, + project=self.owner_project, content=self.owner_content, reason=ReviewReason.BORDERLINE_RELEVANCE, confidence=0.51, ) self.owner_source_config = SourceConfig.objects.create( - tenant=self.owner_tenant, + project=self.owner_project, plugin_name=SourcePluginName.RSS, config={"feed_url": "https://example.com/feed.xml"}, ) @@ -101,10 +106,10 @@ def assert_standardized_validation_error(self, payload, attr): self.assertEqual(payload["type"], "validation_error") self.assertTrue(any(error["attr"] == attr for error in payload["errors"])) - def test_tenant_list_requires_authentication(self): + def test_project_list_requires_authentication(self): self.client.force_authenticate(user=None) - response = self.client.get(reverse("v1:tenant-list"), HTTP_HOST="localhost") + response = self.client.get(reverse("v1:project-list"), HTTP_HOST="localhost") self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) self.assertEqual( @@ -121,28 +126,28 @@ def test_tenant_list_requires_authentication(self): }, ) - def test_tenant_list_is_scoped_to_request_user(self): - response = self.client.get(reverse("v1:tenant-list")) + def test_project_list_is_scoped_to_request_user_groups(self): + response = self.client.get(reverse("v1:project-list")) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(len(response.json()), 1) - self.assertEqual(response.json()[0]["id"], self.owner_tenant.id) + self.assertEqual(response.json()[0]["id"], self.owner_project.id) - def test_entity_list_is_scoped_to_request_user_tenant(self): - response = self.client.get(reverse("v1:tenant-entity-list", kwargs={"tenant_id": self.owner_tenant.id})) + def test_entity_list_is_scoped_to_request_user_project(self): + response = self.client.get(reverse("v1:project-entity-list", kwargs={"project_id": self.owner_project.id})) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(len(response.json()), 1) self.assertEqual(response.json()[0]["id"], self.owner_entity.id) - def test_nested_entity_list_rejects_other_users_tenant(self): - response = self.client.get(reverse("v1:tenant-entity-list", kwargs={"tenant_id": self.other_tenant.id})) + def test_nested_entity_list_rejects_other_users_project(self): + response = self.client.get(reverse("v1:project-entity-list", kwargs={"project_id": self.other_project.id})) self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) def test_feedback_create_assigns_current_user(self): response = self.client.post( - reverse("v1:tenant-feedback-list", kwargs={"tenant_id": self.owner_tenant.id}), + reverse("v1:project-feedback-list", kwargs={"project_id": self.owner_project.id}), { "content": self.owner_content.id, "feedback_type": FeedbackType.UPVOTE, @@ -155,9 +160,9 @@ def test_feedback_create_assigns_current_user(self): self.assertEqual(feedback.user, self.owner) self.assertEqual(feedback.feedback_type, FeedbackType.UPVOTE) - def test_feedback_rejects_cross_tenant_content(self): + def test_feedback_rejects_cross_project_content(self): response = self.client.post( - reverse("v1:tenant-feedback-list", kwargs={"tenant_id": self.owner_tenant.id}), + reverse("v1:project-feedback-list", kwargs={"project_id": self.owner_project.id}), { "content": self.other_content.id, "feedback_type": FeedbackType.DOWNVOTE, @@ -168,9 +173,9 @@ def test_feedback_rejects_cross_tenant_content(self): self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assert_standardized_validation_error(response.json(), "content") - def test_content_create_uses_tenant_from_url(self): + def test_content_create_uses_project_from_url(self): response = self.client.post( - reverse("v1:tenant-content-list", kwargs={"tenant_id": self.owner_tenant.id}), + reverse("v1:project-content-list", kwargs={"project_id": self.owner_project.id}), { "url": "https://example.com/new", "title": "New Content", @@ -179,20 +184,20 @@ def test_content_create_uses_tenant_from_url(self): "source_plugin": "rss", "published_date": "2026-04-22T00:00:00Z", "content_text": "Nested content text", - "tenant": self.other_tenant.id, + "project": self.other_project.id, }, format="json", ) self.assertEqual(response.status_code, status.HTTP_201_CREATED) created_content = Content.objects.get(title="New Content") - self.assertEqual(created_content.tenant, self.owner_tenant) + self.assertEqual(created_content.project, self.owner_project) @patch("core.tasks.run_relevance_scoring_skill.delay") def test_content_skill_action_queues_relevance_scoring(self, run_relevance_scoring_delay_mock): response = self.client.post( - f"/api/v1/tenants/{self.owner_tenant.id}/contents/{self.owner_content.id}/skills/relevance_scoring/", + f"/api/v1/projects/{self.owner_project.id}/contents/{self.owner_content.id}/skills/relevance_scoring/", format="json", ) @@ -214,7 +219,7 @@ def test_content_skill_action_queues_summarization(self, run_summarization_delay self.owner_content.save(update_fields=["relevance_score"]) response = self.client.post( - f"/api/v1/tenants/{self.owner_tenant.id}/contents/{self.owner_content.id}/skills/summarization/", + f"/api/v1/projects/{self.owner_project.id}/contents/{self.owner_content.id}/skills/summarization/", format="json", ) @@ -244,7 +249,7 @@ def test_content_skill_action_runs_find_related(self, search_similar_content_moc ] response = self.client.post( - f"/api/v1/tenants/{self.owner_tenant.id}/contents/{self.owner_content.id}/skills/find_related/", + f"/api/v1/projects/{self.owner_project.id}/contents/{self.owner_content.id}/skills/find_related/", format="json", ) @@ -255,14 +260,14 @@ def test_content_skill_action_runs_find_related(self, search_similar_content_moc def test_authenticated_nested_list_endpoints_smoke(self): list_endpoints = [ - reverse("v1:tenant-config-list", kwargs={"tenant_id": self.owner_tenant.id}), - reverse("v1:tenant-entity-list", kwargs={"tenant_id": self.owner_tenant.id}), - reverse("v1:tenant-content-list", kwargs={"tenant_id": self.owner_tenant.id}), - reverse("v1:tenant-skill-result-list", kwargs={"tenant_id": self.owner_tenant.id}), - reverse("v1:tenant-feedback-list", kwargs={"tenant_id": self.owner_tenant.id}), - reverse("v1:tenant-ingestion-run-list", kwargs={"tenant_id": self.owner_tenant.id}), - reverse("v1:tenant-source-config-list", kwargs={"tenant_id": self.owner_tenant.id}), - reverse("v1:tenant-review-queue-list", kwargs={"tenant_id": self.owner_tenant.id}), + reverse("v1:project-config-list", kwargs={"project_id": self.owner_project.id}), + reverse("v1:project-entity-list", kwargs={"project_id": self.owner_project.id}), + reverse("v1:project-content-list", kwargs={"project_id": self.owner_project.id}), + reverse("v1:project-skill-result-list", kwargs={"project_id": self.owner_project.id}), + reverse("v1:project-feedback-list", kwargs={"project_id": self.owner_project.id}), + reverse("v1:project-ingestion-run-list", kwargs={"project_id": self.owner_project.id}), + reverse("v1:project-source-config-list", kwargs={"project_id": self.owner_project.id}), + reverse("v1:project-review-queue-list", kwargs={"project_id": self.owner_project.id}), ] for endpoint in list_endpoints: @@ -273,45 +278,45 @@ def test_authenticated_nested_list_endpoints_smoke(self): def test_authenticated_nested_detail_endpoints_smoke(self): detail_endpoints = [ reverse( - "v1:tenant-config-detail", - kwargs={"tenant_id": self.owner_tenant.id, "pk": self.owner_config.id}, + "v1:project-config-detail", + kwargs={"project_id": self.owner_project.id, "pk": self.owner_config.id}, ), reverse( - "v1:tenant-entity-detail", - kwargs={"tenant_id": self.owner_tenant.id, "pk": self.owner_entity.id}, + "v1:project-entity-detail", + kwargs={"project_id": self.owner_project.id, "pk": self.owner_entity.id}, ), reverse( - "v1:tenant-content-detail", - kwargs={"tenant_id": self.owner_tenant.id, "pk": self.owner_content.id}, + "v1:project-content-detail", + kwargs={"project_id": self.owner_project.id, "pk": self.owner_content.id}, ), reverse( - "v1:tenant-skill-result-detail", - kwargs={"tenant_id": self.owner_tenant.id, "pk": self.owner_skill_result.id}, + "v1:project-skill-result-detail", + kwargs={"project_id": self.owner_project.id, "pk": self.owner_skill_result.id}, ), reverse( - "v1:tenant-ingestion-run-detail", - kwargs={"tenant_id": self.owner_tenant.id, "pk": self.owner_ingestion_run.id}, + "v1:project-ingestion-run-detail", + kwargs={"project_id": self.owner_project.id, "pk": self.owner_ingestion_run.id}, ), reverse( - "v1:tenant-source-config-detail", - kwargs={"tenant_id": self.owner_tenant.id, "pk": self.owner_source_config.id}, + "v1:project-source-config-detail", + kwargs={"project_id": self.owner_project.id, "pk": self.owner_source_config.id}, ), reverse( - "v1:tenant-review-queue-detail", - kwargs={"tenant_id": self.owner_tenant.id, "pk": self.owner_review_queue.id}, + "v1:project-review-queue-detail", + kwargs={"project_id": self.owner_project.id, "pk": self.owner_review_queue.id}, ), ] feedback = UserFeedback.objects.create( - tenant=self.owner_tenant, + project=self.owner_project, content=self.owner_content, user=self.owner, feedback_type=FeedbackType.UPVOTE, ) detail_endpoints.append( reverse( - "v1:tenant-feedback-detail", - kwargs={"tenant_id": self.owner_tenant.id, "pk": feedback.id}, + "v1:project-feedback-detail", + kwargs={"project_id": self.owner_project.id, "pk": feedback.id}, ) ) @@ -322,7 +327,7 @@ def test_authenticated_nested_detail_endpoints_smoke(self): def test_source_config_create_validates_plugin_config(self): response = self.client.post( - reverse("v1:tenant-source-config-list", kwargs={"tenant_id": self.owner_tenant.id}), + reverse("v1:project-source-config-list", kwargs={"project_id": self.owner_project.id}), {"plugin_name": SourcePluginName.RSS, "config": {}}, format="json", ) diff --git a/core/tests/test_embeddings.py b/core/tests/test_embeddings.py index a1fba3d6..1f003df8 100644 --- a/core/tests/test_embeddings.py +++ b/core/tests/test_embeddings.py @@ -2,6 +2,7 @@ import httpx import pytest +from django.contrib.auth.models import Group from django.core.management import call_command from qdrant_client.http.exceptions import ResponseHandlingException @@ -17,11 +18,11 @@ Content, Entity, IngestionRun, + Project, ReviewQueue, SkillResult, SourceConfig, SourcePluginName, - Tenant, UserFeedback, ) from core.pipeline import ( @@ -43,9 +44,11 @@ def clear_embedding_provider_cache(): @pytest.fixture def embedding_context(django_user_model): user = django_user_model.objects.create_user(username="embed-owner", password="testpass123") - tenant = Tenant.objects.create(name="Embedding Tenant", user=user, topic_description="Infra") + group = Group.objects.create(name="embedding-team") + user.groups.add(group) + project = Project.objects.create(name="Embedding Project", group=group, topic_description="Infra") content = Content.objects.create( - tenant=tenant, + project=project, url="https://example.com/embed", title="Embedding Content", author="Author", @@ -53,7 +56,7 @@ def embedding_context(django_user_model): published_date="2026-04-20T12:00:00Z", content_text="This article covers platform engineering practices.", ) - return SimpleNamespace(user=user, tenant=tenant, content=content) + return SimpleNamespace(user=user, group=group, project=project, content=content) def test_upsert_content_embedding_persists_embedding_id_and_payload(embedding_context, mocker): @@ -73,13 +76,13 @@ def test_upsert_content_embedding_persists_embedding_id_and_payload(embedding_co assert upsert_points[0].payload["content_id"] == embedding_context.content.id assert upsert_points[0].payload["is_reference"] is False -def test_search_similar_returns_qdrant_results_for_tenant_collection(embedding_context, mocker): +def test_search_similar_returns_qdrant_results_for_project_collection(embedding_context, mocker): client_mock = mocker.patch("core.embeddings.get_qdrant_client") scored_point = SimpleNamespace(score=0.91, payload={"content_id": embedding_context.content.id}) client_mock.return_value.get_collection.return_value = SimpleNamespace() client_mock.return_value.search.return_value = [scored_point] - results = search_similar(embedding_context.tenant.id, [0.1, 0.2, 0.3], limit=3, exclude_content_id=embedding_context.content.id) + results = search_similar(embedding_context.project.id, [0.1, 0.2, 0.3], limit=3, exclude_content_id=embedding_context.content.id) assert results == [scored_point] client_mock.return_value.search.assert_called_once() @@ -96,7 +99,7 @@ def test_search_similar_content_embeds_current_content_and_excludes_self(embeddi assert len(results) == 1 embed_text_mock.assert_called_once_with("Embedding Content\n\nThis article covers platform engineering practices.") search_similar_mock.assert_called_once_with( - embedding_context.tenant.id, + embedding_context.project.id, [0.3, 0.2, 0.1], limit=4, is_reference=False, @@ -107,12 +110,12 @@ def test_get_reference_similarity_averages_reference_scores(embedding_context, m search_mock = mocker.patch("core.embeddings.search_similar") search_mock.return_value = [SimpleNamespace(score=0.8), SimpleNamespace(score=0.6)] - similarity = get_reference_similarity(embedding_context.tenant.id, [0.1, 0.2, 0.3]) + similarity = get_reference_similarity(embedding_context.project.id, [0.1, 0.2, 0.3]) assert similarity == 0.7 def test_get_reference_similarity_returns_zero_when_no_reference_matches(embedding_context): - similarity = get_reference_similarity(embedding_context.tenant.id, [0.1, 0.2, 0.3]) + similarity = get_reference_similarity(embedding_context.project.id, [0.1, 0.2, 0.3]) assert similarity == 0.0 @@ -189,17 +192,17 @@ def test_seed_demo_creates_reference_corpus_and_embeds_demo_content(mocker, caps call_command("seed_demo") - tenant = Tenant.objects.get(name="Platform Engineering Weekly") - assert Entity.objects.filter(tenant=tenant).count() == 15 - assert SourceConfig.objects.filter(tenant=tenant).count() == 8 - assert Content.objects.filter(tenant=tenant, is_reference=True).count() == 50 - assert Content.objects.filter(tenant=tenant, is_reference=False).count() == 200 - assert SkillResult.objects.filter(tenant=tenant, skill_name=CLASSIFICATION_SKILL_NAME).count() == 200 - assert SkillResult.objects.filter(tenant=tenant, skill_name=RELEVANCE_SKILL_NAME).count() == 200 - assert SkillResult.objects.filter(tenant=tenant, skill_name=SUMMARIZATION_SKILL_NAME).count() == 115 - assert ReviewQueue.objects.filter(tenant=tenant).exists() - assert UserFeedback.objects.filter(tenant=tenant).count() == 45 - assert IngestionRun.objects.filter(tenant=tenant).count() == 6 + project = Project.objects.get(name="Platform Engineering Weekly") + assert Entity.objects.filter(project=project).count() == 15 + assert SourceConfig.objects.filter(project=project).count() == 8 + assert Content.objects.filter(project=project, is_reference=True).count() == 50 + assert Content.objects.filter(project=project, is_reference=False).count() == 200 + assert SkillResult.objects.filter(project=project, skill_name=CLASSIFICATION_SKILL_NAME).count() == 200 + assert SkillResult.objects.filter(project=project, skill_name=RELEVANCE_SKILL_NAME).count() == 200 + assert SkillResult.objects.filter(project=project, skill_name=SUMMARIZATION_SKILL_NAME).count() == 115 + assert ReviewQueue.objects.filter(project=project).exists() + assert UserFeedback.objects.filter(project=project).count() == 45 + assert IngestionRun.objects.filter(project=project).count() == 6 assert upsert_mock.call_count == 250 output = capsys.readouterr().out assert "Reference corpus items: 50" in output @@ -212,15 +215,15 @@ def test_seed_demo_is_stable_on_rerun(mocker): call_command("seed_demo") call_command("seed_demo") - tenant = Tenant.objects.get(name="Platform Engineering Weekly") - assert Entity.objects.filter(tenant=tenant).count() == 15 - assert SourceConfig.objects.filter(tenant=tenant).count() == 8 - assert Content.objects.filter(tenant=tenant, is_reference=True).count() == 50 - assert Content.objects.filter(tenant=tenant, is_reference=False).count() == 200 - assert SkillResult.objects.filter(tenant=tenant).count() == 515 - assert ReviewQueue.objects.filter(tenant=tenant).count() > 0 - assert UserFeedback.objects.filter(tenant=tenant).count() == 45 - assert IngestionRun.objects.filter(tenant=tenant).count() == 6 + project = Project.objects.get(name="Platform Engineering Weekly") + assert Entity.objects.filter(project=project).count() == 15 + assert SourceConfig.objects.filter(project=project).count() == 8 + assert Content.objects.filter(project=project, is_reference=True).count() == 50 + assert Content.objects.filter(project=project, is_reference=False).count() == 200 + assert SkillResult.objects.filter(project=project).count() == 515 + assert ReviewQueue.objects.filter(project=project).count() > 0 + assert UserFeedback.objects.filter(project=project).count() == 45 + assert IngestionRun.objects.filter(project=project).count() == 6 def test_seed_demo_skips_embeddings_when_vector_stack_is_unavailable(mocker, capsys): @@ -231,10 +234,10 @@ def test_seed_demo_skips_embeddings_when_vector_stack_is_unavailable(mocker, cap call_command("seed_demo") - tenant = Tenant.objects.get(name="Platform Engineering Weekly") - assert Content.objects.filter(tenant=tenant, is_reference=True).count() == 50 - assert Content.objects.filter(tenant=tenant, is_reference=False).count() == 200 - assert SkillResult.objects.filter(tenant=tenant).count() == 515 + project = Project.objects.get(name="Platform Engineering Weekly") + assert Content.objects.filter(project=project, is_reference=True).count() == 50 + assert Content.objects.filter(project=project, is_reference=False).count() == 200 + assert SkillResult.objects.filter(project=project).count() == 515 assert upsert_mock.call_count == 1 combined_output = capsys.readouterr() assert "Skipping remaining embedding sync" in combined_output.err diff --git a/core/tests/test_pipeline.py b/core/tests/test_pipeline.py index 196ccbb1..e081925e 100644 --- a/core/tests/test_pipeline.py +++ b/core/tests/test_pipeline.py @@ -1,8 +1,9 @@ from types import SimpleNamespace import pytest +from django.contrib.auth.models import Group -from core.models import Content, ReviewQueue, ReviewReason, SkillResult, Tenant +from core.models import Content, Project, ReviewQueue, ReviewReason, SkillResult from core.pipeline import CLASSIFICATION_SKILL_NAME, RELEVANCE_SKILL_NAME, SUMMARIZATION_SKILL_NAME from core.tasks import process_content @@ -12,9 +13,11 @@ @pytest.fixture def pipeline_context(django_user_model): user = django_user_model.objects.create_user(username="pipeline-owner", password="testpass123") - tenant = Tenant.objects.create(name="Pipeline Tenant", user=user, topic_description="Platform engineering") + group = Group.objects.create(name="pipeline-team") + user.groups.add(group) + project = Project.objects.create(name="Pipeline Project", group=group, topic_description="Platform engineering") content = Content.objects.create( - tenant=tenant, + project=project, url="https://example.com/article", title="Kubernetes Release Notes", author="Editor", @@ -23,7 +26,7 @@ def pipeline_context(django_user_model): content_text="This article covers a new Kubernetes release and what changed for platform teams.", embedding_id="emb_123", ) - return SimpleNamespace(user=user, tenant=tenant, content=content) + return SimpleNamespace(user=user, group=group, project=project, content=content) def test_process_content_runs_full_pipeline_for_relevant_content(pipeline_context, mocker): @@ -41,7 +44,7 @@ def test_process_content_runs_full_pipeline_for_relevant_content(pipeline_contex "core.pipeline.run_relevance_scoring", return_value={ "relevance_score": 0.92, - "explanation": "Very close to the tenant reference corpus.", + "explanation": "Very close to the project reference corpus.", "used_llm": False, "model_used": "embedding:test", "latency_ms": 0, @@ -84,7 +87,7 @@ def test_process_content_queues_borderline_items_for_review(pipeline_context, mo "core.pipeline.run_relevance_scoring", return_value={ "relevance_score": 0.55, - "explanation": "Borderline similarity to the tenant baseline.", + "explanation": "Borderline similarity to the project baseline.", "used_llm": False, "model_used": "embedding:test", "latency_ms": 0, @@ -117,7 +120,7 @@ def test_process_content_archives_irrelevant_items(pipeline_context, mocker): "core.pipeline.run_relevance_scoring", return_value={ "relevance_score": 0.2, - "explanation": "Far from the tenant reference corpus.", + "explanation": "Far from the project reference corpus.", "used_llm": False, "model_used": "embedding:test", "latency_ms": 0, @@ -149,7 +152,7 @@ def test_process_content_adds_review_item_for_low_confidence_classification(pipe "core.pipeline.run_relevance_scoring", return_value={ "relevance_score": 0.9, - "explanation": "Close to the tenant baseline.", + "explanation": "Close to the project baseline.", "used_llm": False, "model_used": "embedding:test", "latency_ms": 0, diff --git a/core/tests/test_tasks.py b/core/tests/test_tasks.py index 95b91423..d811fa8a 100644 --- a/core/tests/test_tasks.py +++ b/core/tests/test_tasks.py @@ -2,16 +2,17 @@ from types import SimpleNamespace import pytest +from django.contrib.auth.models import Group from core.models import ( Content, Entity, IngestionRun, + Project, RunStatus, SkillStatus, SourceConfig, SourcePluginName, - Tenant, ) from core.pipeline import RELEVANCE_SKILL_NAME, SUMMARIZATION_SKILL_NAME from core.tasks import ( @@ -28,14 +29,16 @@ @pytest.fixture def source_plugin_context(django_user_model): user = django_user_model.objects.create_user(username="plugin-owner", password="testpass123") - tenant = Tenant.objects.create(name="Plugin Tenant", user=user, topic_description="Infra") + group = Group.objects.create(name="plugin-team") + user.groups.add(group) + project = Project.objects.create(name="Plugin Project", group=group, topic_description="Infra") entity = Entity.objects.create( - tenant=tenant, + project=project, name="Example", type="vendor", website_url="https://example.com", ) - return SimpleNamespace(user=user, tenant=tenant, entity=entity) + return SimpleNamespace(user=user, group=group, project=project, entity=entity) def test_run_ingestion_creates_content_from_rss_entries(source_plugin_context, mocker): @@ -43,7 +46,7 @@ def test_run_ingestion_creates_content_from_rss_entries(source_plugin_context, m process_content_delay_mock = mocker.patch("core.tasks.process_content.delay") parse_mock = mocker.patch("core.plugins.rss.feedparser.parse") source_config = SourceConfig.objects.create( - tenant=source_plugin_context.tenant, + project=source_plugin_context.project, plugin_name=SourcePluginName.RSS, config={"feed_url": "https://example.com/feed.xml"}, ) @@ -64,12 +67,12 @@ def test_run_ingestion_creates_content_from_rss_entries(source_plugin_context, m assert result["items_fetched"] == 1 assert result["items_ingested"] == 1 content = Content.objects.get(url="https://example.com/post-1") - assert content.tenant == source_plugin_context.tenant + assert content.project == source_plugin_context.project assert content.entity == source_plugin_context.entity upsert_embedding_mock.assert_called_once_with(content) process_content_delay_mock.assert_called_once_with(content.id) assert SourceConfig.objects.get(pk=source_config.id).last_fetched_at is not None - ingestion_run = IngestionRun.objects.get(tenant=source_plugin_context.tenant, plugin_name=SourcePluginName.RSS) + ingestion_run = IngestionRun.objects.get(project=source_plugin_context.project, plugin_name=SourcePluginName.RSS) assert ingestion_run.status == RunStatus.SUCCESS def test_run_ingestion_skips_duplicate_urls(source_plugin_context, mocker): @@ -77,12 +80,12 @@ def test_run_ingestion_skips_duplicate_urls(source_plugin_context, mocker): process_content_delay_mock = mocker.patch("core.tasks.process_content.delay") parse_mock = mocker.patch("core.plugins.rss.feedparser.parse") source_config = SourceConfig.objects.create( - tenant=source_plugin_context.tenant, + project=source_plugin_context.project, plugin_name=SourcePluginName.RSS, config={"feed_url": "https://example.com/feed.xml"}, ) Content.objects.create( - tenant=source_plugin_context.tenant, + project=source_plugin_context.project, entity=source_plugin_context.entity, url="https://example.com/post-1", title="Existing", @@ -116,7 +119,7 @@ def test_run_ingestion_creates_content_from_reddit_posts(source_plugin_context, process_content_delay_mock = mocker.patch("core.tasks.process_content.delay") reddit_mock = mocker.patch("core.plugins.reddit.praw.Reddit") source_config = SourceConfig.objects.create( - tenant=source_plugin_context.tenant, + project=source_plugin_context.project, plugin_name=SourcePluginName.REDDIT, config={"subreddit": "python", "listing": "new", "limit": 5}, ) @@ -145,17 +148,17 @@ def test_run_ingestion_creates_content_from_reddit_posts(source_plugin_context, def test_run_all_ingestions_enqueues_active_source_configs(source_plugin_context, mocker): delay_mock = mocker.patch("core.tasks.run_ingestion.delay") active_one = SourceConfig.objects.create( - tenant=source_plugin_context.tenant, + project=source_plugin_context.project, plugin_name=SourcePluginName.RSS, config={"feed_url": "https://example.com/feed.xml"}, ) active_two = SourceConfig.objects.create( - tenant=source_plugin_context.tenant, + project=source_plugin_context.project, plugin_name=SourcePluginName.REDDIT, config={"subreddit": "python"}, ) SourceConfig.objects.create( - tenant=source_plugin_context.tenant, + project=source_plugin_context.project, plugin_name=SourcePluginName.RSS, config={"feed_url": "https://example.com/inactive.xml"}, is_active=False, @@ -171,7 +174,7 @@ def test_run_all_ingestions_enqueues_active_source_configs(source_plugin_context def test_run_ingestion_marks_failure_when_plugin_errors(source_plugin_context, mocker): parse_mock = mocker.patch("core.plugins.rss.feedparser.parse") source_config = SourceConfig.objects.create( - tenant=source_plugin_context.tenant, + project=source_plugin_context.project, plugin_name=SourcePluginName.RSS, config={"feed_url": "https://example.com/feed.xml"}, ) @@ -180,14 +183,14 @@ def test_run_ingestion_marks_failure_when_plugin_errors(source_plugin_context, m with pytest.raises(RuntimeError, match="feed unavailable"): run_ingestion(source_config.id) - ingestion_run = IngestionRun.objects.get(tenant=source_plugin_context.tenant, plugin_name=SourcePluginName.RSS) + ingestion_run = IngestionRun.objects.get(project=source_plugin_context.project, plugin_name=SourcePluginName.RSS) assert ingestion_run.status == RunStatus.FAILED assert ingestion_run.error_message == "feed unavailable" def test_queue_content_skill_enqueues_relevance_task(source_plugin_context, mocker): content = Content.objects.create( - tenant=source_plugin_context.tenant, + project=source_plugin_context.project, entity=source_plugin_context.entity, url="https://example.com/manual-content", title="Manual Content", @@ -206,7 +209,7 @@ def test_queue_content_skill_enqueues_relevance_task(source_plugin_context, mock def test_run_relevance_scoring_skill_updates_pending_result(source_plugin_context, mocker): content = Content.objects.create( - tenant=source_plugin_context.tenant, + project=source_plugin_context.project, entity=source_plugin_context.entity, url="https://example.com/relevance-content", title="Relevance Content", @@ -219,7 +222,7 @@ def test_run_relevance_scoring_skill_updates_pending_result(source_plugin_contex "core.pipeline.run_relevance_scoring", return_value={ "relevance_score": 0.82, - "explanation": "Strong match for the tenant topic.", + "explanation": "Strong match for the project topic.", "used_llm": False, "model_used": "embedding:test", "latency_ms": 0, @@ -242,7 +245,7 @@ def test_run_relevance_scoring_skill_updates_pending_result(source_plugin_contex def test_run_summarization_skill_marks_result_failed_when_relevance_is_too_low(source_plugin_context, mocker): content = Content.objects.create( - tenant=source_plugin_context.tenant, + project=source_plugin_context.project, entity=source_plugin_context.entity, url="https://example.com/summary-content", title="Summary Content", diff --git a/core/utils.py b/core/utils.py index 69800201..817a1410 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1,11 +1,11 @@ from django.db.models import Avg -from .models import TenantConfig +from .models import ProjectConfig def dashboard_callback(request, context): - # Calculate the average authority weight across all tenants - avg_weight = TenantConfig.objects.aggregate(Avg('upvote_authority_weight'))['upvote_authority_weight__avg'] + # Calculate the average authority weight across all projects. + avg_weight = ProjectConfig.objects.aggregate(Avg("upvote_authority_weight"))["upvote_authority_weight__avg"] # Add it to the template context context.update({ diff --git a/frontend/app/admin/health/page.tsx b/frontend/app/admin/health/page.tsx index b88264e8..7d90464f 100644 --- a/frontend/app/admin/health/page.tsx +++ b/frontend/app/admin/health/page.tsx @@ -1,12 +1,12 @@ import { AppShell } from "@/components/app-shell" import { StatusBadge } from "@/components/status-badge" import { - getTenantIngestionRuns, - getTenants, - getTenantSourceConfigs, + getProjectIngestionRuns, + getProjects, + getProjectSourceConfigs, } from "@/lib/api" import type { HealthStatus } from "@/lib/types" -import { formatDate, healthTone, selectTenant } from "@/lib/view-helpers" +import { formatDate, healthTone, selectProject } from "@/lib/view-helpers" type HealthPageProps = { searchParams: Promise> @@ -40,27 +40,27 @@ function deriveSourceStatus( export default async function HealthPage({ searchParams }: HealthPageProps) { const resolvedSearchParams = await searchParams - const tenants = await getTenants() - const selectedTenant = selectTenant(tenants, resolvedSearchParams) + const projects = await getProjects() + const selectedProject = selectProject(projects, resolvedSearchParams) - if (!selectedTenant) { + if (!selectedProject) { return (
- Create a tenant first in Django admin. + Create a project first in Django admin.
) } const [sourceConfigs, ingestionRuns] = await Promise.all([ - getTenantSourceConfigs(selectedTenant.id), - getTenantIngestionRuns(selectedTenant.id), + getProjectSourceConfigs(selectedProject.id), + getProjectIngestionRuns(selectedProject.id), ]) const latestRunByPlugin = new Map() @@ -74,8 +74,8 @@ export default async function HealthPage({ searchParams }: HealthPageProps) {
@@ -95,7 +95,7 @@ export default async function HealthPage({ searchParams }: HealthPageProps) {
- No source configurations exist for this tenant yet. + No source configurations exist for this project yet.
diff --git a/frontend/app/admin/sources/page.tsx b/frontend/app/admin/sources/page.tsx index 82d2c54c..4acdb254 100644 --- a/frontend/app/admin/sources/page.tsx +++ b/frontend/app/admin/sources/page.tsx @@ -1,15 +1,15 @@ import { AppShell } from "@/components/app-shell" import { StatusBadge } from "@/components/status-badge" import { - getTenantIngestionRuns, - getTenants, - getTenantSourceConfigs, + getProjectIngestionRuns, + getProjects, + getProjectSourceConfigs, } from "@/lib/api" import { formatDate, getErrorMessage, getSuccessMessage, - selectTenant, + selectProject, } from "@/lib/view-helpers" type SourcesPageProps = { @@ -33,27 +33,27 @@ const primaryButtonClass = export default async function SourcesPage({ searchParams }: SourcesPageProps) { const resolvedSearchParams = await searchParams - const tenants = await getTenants() - const selectedTenant = selectTenant(tenants, resolvedSearchParams) + const projects = await getProjects() + const selectedProject = selectProject(projects, resolvedSearchParams) - if (!selectedTenant) { + if (!selectedProject) { return (
- Create a tenant first in Django admin. + Create a project first in Django admin.
) } const [sourceConfigs, ingestionRuns] = await Promise.all([ - getTenantSourceConfigs(selectedTenant.id), - getTenantIngestionRuns(selectedTenant.id), + getProjectSourceConfigs(selectedProject.id), + getProjectIngestionRuns(selectedProject.id), ]) const latestRunByPlugin = new Map() for (const ingestionRun of ingestionRuns) { @@ -69,8 +69,8 @@ export default async function SourcesPage({ searchParams }: SourcesPageProps) { {errorMessage ? (
{errorMessage}
@@ -87,11 +87,11 @@ export default async function SourcesPage({ searchParams }: SourcesPageProps) { action="/api/source-configs" method="POST" > - +