Schedule live telemetry quality checks

.github/workflows/posthog-quality.yml

@@ -0,0 +1,48 @@
+name: PostHog telemetry quality
+
+on:
+  workflow_dispatch:
+    inputs:
+      days:
+        description: Days of live events to inspect
+        required: false
+        default: '7'
+      limit_per_event:
+        description: Maximum events to sample per contracted event
+        required: false
+        default: '100'
+  schedule:
+    - cron: '23 14 * * *'
+
+concurrency:
+  group: posthog-telemetry-quality
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  live-quality:
+    name: Live telemetry contract and coverage
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6.0.2
+      - uses: actions/setup-node@v6.3.0
+        with:
+          node-version: 22
+          cache: npm
+      - run: npm ci
+      - name: Run live telemetry quality check
+        env:
+          POSTHOG_PERSONAL_API_KEY: ${{ secrets.POSTHOG_PERSONAL_API_KEY }}
+          POSTHOG_HOST: https://us.i.posthog.com
+          POSTHOG_PROJECT_ID: ${{ secrets.POSTHOG_PROJECT_ID }}
+          QUALITY_DAYS: ${{ github.event.inputs.days || '7' }}
+          QUALITY_LIMIT_PER_EVENT: ${{ github.event.inputs.limit_per_event || '100' }}
+        run: |
+          set -euo pipefail
+          if [ -z "${POSTHOG_PERSONAL_API_KEY:-}" ] || [ -z "${POSTHOG_PROJECT_ID:-}" ]; then
+            echo "::error::POSTHOG_PERSONAL_API_KEY and POSTHOG_PROJECT_ID Actions secrets are required."
+            exit 1
+          fi
+          npx nx run posthog-tools:quality:live -- --days "$QUALITY_DAYS" --limit-per-event "$QUALITY_LIMIT_PER_EVENT" --require-critical-coverage

tools/posthog/README.md

@@ -52,11 +52,11 @@ Requires a **Personal API Key** with `dashboard:write`, `insight:write`, `cohort
 
 Env vars (see `.env.example` at repo root):
 
-| Variable | Purpose |
-|----------|---------|
-| `POSTHOG_PERSONAL_API_KEY` | Personal API Key (Bearer) |
-| `POSTHOG_HOST` | `https://us.i.posthog.com` (default) or your region |
-| `POSTHOG_PROJECT_ID` | Numeric project id (visible in PostHog URL) |
+| Variable                   | Purpose                                             |
+| -------------------------- | --------------------------------------------------- |
+| `POSTHOG_PERSONAL_API_KEY` | Personal API Key (Bearer)                           |
+| `POSTHOG_HOST`             | `https://us.i.posthog.com` (default) or your region |
+| `POSTHOG_PROJECT_ID`       | Numeric project id (visible in PostHog URL)         |
 
 **CI** uses the same key (write-scoped) for `--plan` only. **Production hardening TODO:** create a read-only Personal API Key for CI and add it as `POSTHOG_PERSONAL_API_KEY_READONLY` in GitHub Actions secrets. Local development continues using the write-scoped key for `--apply` and `--report`.
 
@@ -65,8 +65,8 @@ Env vars (see `.env.example` at repo root):
 ```jsonc
 // tools/posthog/dashboards/developer-funnel.json
 {
-  "slug": "developer-funnel",                  // local id, stable across syncs
-  "posthog_id": null,                           // assigned on first sync; do not edit
+  "slug": "developer-funnel", // local id, stable across syncs
+  "posthog_id": null, // assigned on first sync; do not edit
   "name": "GTM · Developer funnel",
   "description": "Pageview → install → cockpit activation.",
   "tags": ["gtm", "developer-track"],
@@ -99,6 +99,8 @@ Event names must match [`docs/gtm/taxonomy.md`](../../docs/gtm/taxonomy.md). The
 
 - `taxonomy.spec.ts` and `telemetry-contract.spec.ts` guard committed dashboard JSON against undocumented events, unsupported breakdowns, unsupported filters, runtime dashboard coverage drift, and forbidden sensitive runtime fields.
 - `npm run posthog:quality -- --days 7 --limit-per-event 25` samples recent live PostHog events and validates observed payloads against the same contract. It exits non-zero for missing required properties or forbidden sensitive properties, and prints warnings for non-contract fields.
+- `npm run posthog:quality -- --days 7 --limit-per-event 100 --require-critical-coverage` also requires recent samples for critical install and runtime events. The scheduled `PostHog telemetry quality` workflow runs this thresholded check daily and supports manual dispatch.
+- The live workflow requires Actions secrets named `POSTHOG_PERSONAL_API_KEY` and `POSTHOG_PROJECT_ID`.
 
 ## Sync semantics
 

tools/posthog/live-quality.spec.ts

@@ -2,9 +2,11 @@ import { test } from 'node:test';
 import assert from 'node:assert/strict';
 import {
   analyzeTelemetryEvents,
+  analyzeTelemetryCoverage,
   fetchRecentContractEvents,
   formatLiveQualityReport,
   hasBlockingFindings,
+  type LiveCoverageRequirement,
   type LiveTelemetryEvent,
 } from './live-quality.js';
 import { TELEMETRY_EVENT_CONTRACT } from './telemetry-contract.js';
@@ -38,7 +40,7 @@ test('analyzeTelemetryEvents flags missing required and forbidden properties', (
         property: 'messages',
         kind: 'forbidden_property',
       },
-    ],
+    ]
   );
   assert.equal(hasBlockingFindings(findings), true);
 });
@@ -69,11 +71,29 @@ test('analyzeTelemetryEvents warns on non-contract properties but ignores PostHo
         property: 'accidental_extra',
         kind: 'unexpected_property',
       },
-    ],
+    ]
   );
   assert.equal(hasBlockingFindings(findings), false);
 });
 
+test('analyzeTelemetryEvents ignores PostHog attribution metadata', () => {
+  const findings = analyzeTelemetryEvents([
+    {
+      event: 'marketing:cta_click',
+      timestamp: '2026-05-17T00:00:00Z',
+      properties: {
+        cta_id: 'hero_docs',
+        gclid: 'click-id',
+        fbclid: 'click-id',
+        utm_source: 'newsletter',
+        utm_campaign: 'launch',
+      },
+    },
+  ]);
+
+  assert.deepEqual(findings, []);
+});
+
 test('formatLiveQualityReport summarizes clean coverage and warnings', () => {
   const events: LiveTelemetryEvent[] = [
     {
@@ -98,6 +118,69 @@ test('formatLiveQualityReport summarizes clean coverage and warnings', () => {
   assert.match(report, /unexpected/);
 });
 
+test('analyzeTelemetryCoverage flags required events with no recent samples', () => {
+  const requirements: LiveCoverageRequirement[] = [
+    { event: 'ngaf:runtime_request_created', minCount: 1 },
+    { event: 'ngaf:stream_started', minCount: 2 },
+  ];
+
+  const findings = analyzeTelemetryCoverage(
+    [
+      {
+        event: 'ngaf:stream_started',
+        timestamp: '2026-05-17T00:00:00Z',
+        properties: { transport: 'langgraph' },
+      },
+    ],
+    requirements
+  );
+
+  assert.deepEqual(
+    findings.map((finding) => ({
+      severity: finding.severity,
+      event: finding.event,
+      property: finding.property,
+      kind: finding.kind,
+      message: finding.message,
+    })),
+    [
+      {
+        severity: 'error',
+        event: 'ngaf:runtime_request_created',
+        property: 'event_count',
+        kind: 'insufficient_event_coverage',
+        message:
+          'ngaf:runtime_request_created has 0 recent events; expected at least 1',
+      },
+      {
+        severity: 'error',
+        event: 'ngaf:stream_started',
+        property: 'event_count',
+        kind: 'insufficient_event_coverage',
+        message: 'ngaf:stream_started has 1 recent event; expected at least 2',
+      },
+    ]
+  );
+  assert.equal(hasBlockingFindings(findings), true);
+});
+
+test('formatLiveQualityReport includes coverage requirements', () => {
+  const report = formatLiveQualityReport({
+    days: 7,
+    events: [],
+    findings: analyzeTelemetryCoverage(
+      [],
+      [{ event: 'ngaf:postinstall', minCount: 1 }]
+    ),
+    checkedEvents: ['ngaf:postinstall'],
+    coverageRequirements: [{ event: 'ngaf:postinstall', minCount: 1 }],
+  });
+
+  assert.match(report, /\| Event \| Sampled events \| Required minimum \|/);
+  assert.match(report, /\| ngaf:postinstall \| 0 \| 1 \|/);
+  assert.match(report, /insufficient_event_coverage/);
+});
+
 test('fetchRecentContractEvents requests each contract event with bounded limits', async () => {
   const calls: unknown[] = [];
   const client = {
@@ -161,9 +244,17 @@ test('every contracted event can be analyzed without a bespoke case', () => {
     event,
     timestamp: '2026-05-17T00:00:00Z',
     properties: Object.fromEntries(
-      TELEMETRY_EVENT_CONTRACT[event].requiredProperties.map((property) => [property, 'x']),
+      TELEMETRY_EVENT_CONTRACT[event].requiredProperties.map((property) => [
+        property,
+        'x',
+      ])
     ),
   }));
 
-  assert.equal(analyzeTelemetryEvents(events).some((finding) => finding.severity === 'error'), false);
+  assert.equal(
+    analyzeTelemetryEvents(events).some(
+      (finding) => finding.severity === 'error'
+    ),
+    false
+  );
 });