Add source diversity analysis WP

Author: webstackdev

core/templates/admin/source_diversity_snapshot_changelist_widget.html

@@ -0,0 +1,64 @@
+{% if dashboard_stats or source_diversity_alerts or source_diversity_project_drilldowns %}
+    <div class="mb-6 flex flex-col gap-4">
+        {% if dashboard_stats %}
+            <div class="grid grid-cols-1 gap-4 md:grid-cols-2 xl:grid-cols-4">
+                {% for stat in dashboard_stats %}
+                    <div class="rounded-default border border-base-200 bg-white px-4 py-4 shadow-xs dark:border-base-800 dark:bg-base-900">
+                        <div class="flex items-center justify-between gap-3">
+                            <div>
+                                <p class="text-xs font-medium uppercase tracking-tight text-base-500 dark:text-base-400">{{ stat.title }}</p>
+                                <p class="mt-2 text-2xl font-semibold text-font-important-light dark:text-font-important-dark">{{ stat.value }}</p>
+                            </div>
+                            <span class="material-symbols-outlined text-base-400 dark:text-base-500">{{ stat.icon }}</span>
+                        </div>
+                    </div>
+                {% endfor %}
+            </div>
+        {% endif %}
+        {% if source_diversity_alerts %}
+            <div class="rounded-default border border-base-200 bg-white px-4 py-4 shadow-xs dark:border-base-800 dark:bg-base-900">
+                <h2 class="text-sm font-semibold text-font-important-light dark:text-font-important-dark">Concentration Alerts</h2>
+                <div class="mt-4 grid grid-cols-1 gap-3 xl:grid-cols-2">
+                    {% for alert in source_diversity_alerts %}
+                        <div class="rounded-default border border-orange-200 bg-orange-50 px-4 py-3 text-sm text-orange-800 dark:border-orange-500/30 dark:bg-orange-500/10 dark:text-orange-300">
+                            {{ alert.message }}
+                        </div>
+                    {% endfor %}
+                </div>
+            </div>
+        {% endif %}
+        {% if source_diversity_project_drilldowns %}
+            <div class="rounded-default border border-base-200 bg-white px-4 py-4 shadow-xs dark:border-base-800 dark:bg-base-900">
+                <div>
+                    <h2 class="text-sm font-semibold text-font-important-light dark:text-font-important-dark">Project Drilldown</h2>
+                    <p class="mt-1 text-sm text-base-500 dark:text-base-400">
+                        Jump into filtered source diversity history for a single project.
+                    </p>
+                </div>
+                <div class="mt-4 grid grid-cols-1 gap-3 xl:grid-cols-2">
+                    {% for drilldown in source_diversity_project_drilldowns %}
+                        <a href="{{ drilldown.href }}"
+                           class="rounded-default border border-base-200 px-4 py-3 transition hover:border-primary-600 hover:bg-base-50 dark:border-base-800 dark:hover:border-primary-500 dark:hover:bg-base-800">
+                            <div class="flex items-start justify-between gap-3">
+                                <div>
+                                    <p class="font-medium text-font-important-light dark:text-font-important-dark">{{ drilldown.project_name }}</p>
+                                    <p class="mt-1 text-sm text-base-500 dark:text-base-400">
+                                        {{ drilldown.snapshot_count }} snapshot{{ drilldown.snapshot_count|pluralize }} · {{ drilldown.latest_snapshot }}
+                                    </p>
+                                </div>
+                                <span class="rounded-full bg-base-100 px-2 py-1 text-xs font-medium text-base-700 dark:bg-base-800 dark:text-base-300">
+                                    {{ drilldown.alert_count }} alert{{ drilldown.alert_count|pluralize }}
+                                </span>
+                            </div>
+                            <div class="mt-3 flex flex-wrap gap-x-4 gap-y-1 text-sm text-base-600 dark:text-base-300">
+                                <span>Plugin {{ drilldown.plugin_entropy }}</span>
+                                <span>Source {{ drilldown.source_entropy }}</span>
+                                <span>Top plugin {{ drilldown.top_plugin_share }}</span>
+                            </div>
+                        </a>
+                    {% endfor %}
+                </div>
+            </div>
+        {% endif %}
+    </div>
+{% endif %}

ingestion/tasks.py

@@ -4,7 +4,7 @@
 
 from celery import shared_task
 from django.conf import settings
-from django.db.models import Q
+from django.db.models import Model, Q
 from django.utils import timezone
 
 from content.deduplication import canonicalize_url
@@ -16,6 +16,17 @@
 logger = logging.getLogger(__name__)
 
 
+def _require_pk(instance: Model) -> int:
+    """Return a saved model primary key as an ``int``."""
+
+    instance_pk = instance.pk
+    if instance_pk is None:
+        raise ValueError(
+            f"{instance.__class__.__name__} must be saved before task dispatch"
+        )
+    return int(instance_pk)
+
+
 @shared_task(name="core.tasks.run_ingestion")
 def run_ingestion(source_config_id: int):
     """Fetch new content for one source config and record an ingestion run."""
@@ -74,7 +85,10 @@ def _ingest_source_config(source_config: SourceConfig) -> tuple[int, int]:
     for item in fetched_items:
         if _content_exists_for_item(source_config, item):
             continue
-        source_metadata = getattr(item, "source_metadata", None) or {}
+        source_metadata = {
+            **(getattr(item, "source_metadata", None) or {}),
+            "source_config_id": _require_pk(source_config),
+        }
         content = Content.objects.create(
             project=source_config.project,
             entity=_match_entity_for_item(plugin, item),

trends/admin.py

@@ -8,7 +8,7 @@
 from django.urls import reverse
 from django.utils import timezone
 
-from trends.models import TopicCentroidSnapshot
+from trends.models import SourceDiversitySnapshot, TopicCentroidSnapshot
 
 
 def _project_pk(snapshot: TopicCentroidSnapshot) -> int:
@@ -44,6 +44,32 @@ def _drift_card_color(value) -> str:
     return "danger"
 
 
+def _diversity_card_color(value) -> str:
+    """Return an admin card severity for normalized diversity scores."""
+
+    if value is None:
+        return "info"
+    numeric_value = float(value)
+    if numeric_value >= 0.75:
+        return "success"
+    if numeric_value >= 0.4:
+        return "warning"
+    return "danger"
+
+
+def _share_card_color(value) -> str:
+    """Return an admin card severity for concentration share metrics."""
+
+    if value is None:
+        return "info"
+    numeric_value = float(value)
+    if numeric_value <= 0.4:
+        return "success"
+    if numeric_value <= 0.7:
+        return "warning"
+    return "danger"
+
+
 def _format_snapshot_freshness(computed_at) -> str:
     """Return a compact human-readable age for the latest snapshot."""
 
@@ -115,6 +141,46 @@ def _build_topic_centroid_project_drilldowns(queryset, changelist_url: str):
     return project_drilldowns
 
 
+def _build_source_diversity_project_drilldowns(queryset, changelist_url: str):
+    """Build one filtered-history drilldown row per project."""
+
+    latest_by_project: dict[int, SourceDiversitySnapshot] = {}
+    snapshot_counts: dict[int, int] = {}
+    ordered_snapshots = queryset.select_related("project").order_by(
+        "project_id", "-computed_at"
+    )
+
+    for snapshot in ordered_snapshots:
+        project_id = int(snapshot.project_id)
+        snapshot_counts[project_id] = snapshot_counts.get(project_id, 0) + 1
+        latest_by_project.setdefault(project_id, snapshot)
+
+    project_drilldowns = []
+    for snapshot in sorted(
+        latest_by_project.values(),
+        key=lambda value: value.project.name.lower(),
+    ):
+        project_id = int(snapshot.project_id)
+        alerts = cast(
+            list[dict[str, Any]], (snapshot.breakdown or {}).get("alerts", [])
+        )
+        project_drilldowns.append(
+            {
+                "project_id": project_id,
+                "project_name": snapshot.project.name,
+                "snapshot_count": snapshot_counts[project_id],
+                "latest_snapshot": _format_snapshot_freshness(snapshot.computed_at),
+                "plugin_entropy": f"{_score_to_percent(snapshot.plugin_entropy):.1f}%",
+                "source_entropy": f"{_score_to_percent(snapshot.source_entropy):.1f}%",
+                "top_plugin_share": f"{_score_to_percent(snapshot.top_plugin_share):.1f}%",
+                "alert_count": len(alerts),
+                "href": f"{changelist_url}?{urlencode({'project__id__exact': project_id})}",
+            }
+        )
+
+    return project_drilldowns
+
+
 @admin.register(TopicCentroidSnapshot)
 class TopicCentroidSnapshotAdmin(admin.ModelAdmin):
     """Admin view for persisted topic-centroid history and drift."""
@@ -219,3 +285,120 @@ def changelist_view(self, request, extra_context=None):
             _build_topic_centroid_project_drilldowns(queryset, changelist_url)
         )
         return super().changelist_view(request, extra_context=extra_context)
+
+
+@admin.register(SourceDiversitySnapshot)
+class SourceDiversitySnapshotAdmin(admin.ModelAdmin):
+    """Admin view for persisted source-diversity history and concentration alerts."""
+
+    list_before_template = "admin/source_diversity_snapshot_changelist_widget.html"
+    list_display = (
+        "project",
+        "display_plugin_entropy",
+        "display_source_entropy",
+        "display_author_entropy",
+        "display_top_plugin_share",
+        "computed_at",
+    )
+    list_filter = (
+        "window_days",
+        ("project", admin.RelatedOnlyFieldListFilter),
+        "computed_at",
+    )
+    search_fields = ("project__name",)
+    autocomplete_fields = ("project",)
+
+    @admin.display(description="Plugin Diversity", ordering="plugin_entropy")
+    def display_plugin_entropy(self, obj):
+        """Render normalized plugin diversity as a percentage."""
+
+        return f"{_score_to_percent(obj.plugin_entropy):.1f}%"
+
+    @admin.display(description="Source Diversity", ordering="source_entropy")
+    def display_source_entropy(self, obj):
+        """Render normalized source diversity as a percentage."""
+
+        return f"{_score_to_percent(obj.source_entropy):.1f}%"
+
+    @admin.display(description="Author Diversity", ordering="author_entropy")
+    def display_author_entropy(self, obj):
+        """Render normalized author diversity as a percentage."""
+
+        return f"{_score_to_percent(obj.author_entropy):.1f}%"
+
+    @admin.display(description="Top Plugin Share", ordering="top_plugin_share")
+    def display_top_plugin_share(self, obj):
+        """Render the largest plugin share as a percentage."""
+
+        return f"{_score_to_percent(obj.top_plugin_share):.1f}%"
+
+    def changelist_view(self, request, extra_context=None):
+        """Augment the changelist with diversity summaries and alert callouts."""
+
+        queryset = self.get_queryset(request)
+        changelist_url = reverse(
+            f"{self.admin_site.name}:{self.model._meta.app_label}_{self.model._meta.model_name}_changelist"
+        )
+        metrics = queryset.aggregate(
+            avg_plugin_entropy=Avg("plugin_entropy"),
+            avg_source_entropy=Avg("source_entropy"),
+            avg_author_entropy=Avg("author_entropy"),
+            avg_top_plugin_share=Avg("top_plugin_share"),
+            latest_snapshot_at=Max("computed_at"),
+        )
+        alerts = [
+            alert
+            for snapshot in queryset.order_by("project_id", "-computed_at")
+            for alert in cast(
+                list[dict[str, Any]], (snapshot.breakdown or {}).get("alerts", [])
+            )
+        ]
+
+        extra_context = cast(dict[str, Any], extra_context or {})
+        extra_context["dashboard_stats"] = [
+            {
+                "title": "Plugin Diversity",
+                "value": (
+                    f"{_score_to_percent(metrics['avg_plugin_entropy']):.1f}%"
+                    if metrics["avg_plugin_entropy"] is not None
+                    else "-"
+                ),
+                "icon": "hub",
+                "color": _diversity_card_color(metrics["avg_plugin_entropy"]),
+            },
+            {
+                "title": "Source Diversity",
+                "value": (
+                    f"{_score_to_percent(metrics['avg_source_entropy']):.1f}%"
+                    if metrics["avg_source_entropy"] is not None
+                    else "-"
+                ),
+                "icon": "lan",
+                "color": _diversity_card_color(metrics["avg_source_entropy"]),
+            },
+            {
+                "title": "Author Diversity",
+                "value": (
+                    f"{_score_to_percent(metrics['avg_author_entropy']):.1f}%"
+                    if metrics["avg_author_entropy"] is not None
+                    else "-"
+                ),
+                "icon": "group",
+                "color": _diversity_card_color(metrics["avg_author_entropy"]),
+            },
+            {
+                "title": "Top Plugin Share",
+                "value": (
+                    f"{_score_to_percent(metrics['avg_top_plugin_share']):.1f}%"
+                    if metrics["avg_top_plugin_share"] is not None
+                    else "-"
+                ),
+                "icon": "pie_chart",
+                "color": _share_card_color(metrics["avg_top_plugin_share"]),
+            },
+        ]
+        extra_context["source_diversity_alerts"] = alerts
+        extra_context["source_diversity_project_drilldowns"] = (
+            _build_source_diversity_project_drilldowns(queryset, changelist_url)
+        )
+        return super().changelist_view(request, extra_context=extra_context)

trends/api.py

@@ -19,13 +19,16 @@
 from core.permissions import IsProjectContributor, IsProjectMember
 from trends.models import (
     ContentClusterMembership,
+    SourceDiversitySnapshot,
     ThemeSuggestion,
     ThemeSuggestionStatus,
     TopicCentroidSnapshot,
     TopicCluster,
     TopicVelocitySnapshot,
 )
 from trends.serializers import (
+    SourceDiversityObservabilitySummarySerializer,
+    SourceDiversitySnapshotSerializer,
     ThemeSuggestionDismissSerializer,
     ThemeSuggestionSerializer,
     TopicClusterDetailSerializer,
@@ -334,3 +337,56 @@ def summary(self, request, *args, **kwargs):
             context=self.get_serializer_context(),
         )
         return Response(serializer.data)
+
+
+@document_project_owned_viewset(
+    resource_plural="source diversity snapshots",
+    resource_singular="source diversity snapshot",
+    create_description="Source diversity snapshots are pipeline-managed observability rows and are exposed read-only for health analysis.",
+    tag="Observability",
+    action_overrides=build_crud_action_overrides(
+        SourceDiversitySnapshotSerializer,
+        resource_plural="source diversity snapshots for the selected project",
+        resource_singular="source diversity snapshot",
+    ),
+)
+class SourceDiversitySnapshotViewSet(
+    ProjectOwnedQuerysetMixin, viewsets.ReadOnlyModelViewSet
+):
+    """Inspect persisted source-diversity history for a project."""
+
+    serializer_class = SourceDiversitySnapshotSerializer
+    queryset = SourceDiversitySnapshot.objects.select_related("project")
+
+    def get_permissions(self):
+        """Restrict source-diversity observability to project contributors."""
+
+        return [IsProjectContributor()]
+
+    @extend_schema(
+        summary="Get source diversity summary",
+        description=(
+            "Return the latest persisted source-diversity snapshot for the selected project "
+            "along with the number of stored snapshots."
+        ),
+        request=None,
+        responses={
+            200: SourceDiversityObservabilitySummarySerializer,
+            403: AUTHENTICATION_REQUIRED_RESPONSE,
+        },
+        tags=["Observability"],
+    )
+    @action(detail=False, methods=["get"], url_path="summary")
+    def summary(self, request, *args, **kwargs):
+        """Return source-diversity summary metrics for the current project."""
+
+        queryset = self.get_queryset()
+        serializer = SourceDiversityObservabilitySummarySerializer(
+            {
+                "project": _require_pk(self.get_project()),
+                "snapshot_count": queryset.count(),
+                "latest_snapshot": queryset.order_by("-computed_at").first(),
+            },
+            context=self.get_serializer_context(),
+        )
+        return Response(serializer.data)