Add live telemetry quality checks

package.json

@@ -12,6 +12,7 @@
     "posthog:sync": "nx run posthog-tools:sync:plan",
     "posthog:apply": "nx run posthog-tools:sync:apply",
     "posthog:report": "nx run posthog-tools:report",
+    "posthog:quality": "nx run posthog-tools:quality:live",
     "posthog:generate-types": "nx run posthog-tools:generate-types",
     "telemetry:install-smoke": "node libs/telemetry/scripts/smoke-install-telemetry.mjs dist/libs/chat dist/libs/langgraph dist/libs/ag-ui dist/libs/render dist/libs/a2ui dist/libs/licensing dist/libs/telemetry"
   },

tools/posthog/README.md

@@ -14,6 +14,7 @@ tools/posthog/
 ├── schema.ts                       # zod schemas for local JSON
 ├── sync.ts                         # CLI: plan / apply / writeback
 ├── report.ts                       # CLI: pull insights → markdown
+├── live-quality.ts                 # CLI: sample recent events and validate payload quality
 ├── *.spec.ts                       # tests
 ├── types/posthog-api.gen.ts        # generated from PostHog OpenAPI spec
 ├── scripts/generate-types.ts       # regenerate the above
@@ -30,6 +31,7 @@ All commands wrap `nx run posthog-tools:*`. Root-package aliases:
 npm run posthog:sync       # → nx run posthog-tools:sync:plan
 npm run posthog:apply      # → nx run posthog-tools:sync:apply
 npm run posthog:report     # → nx run posthog-tools:report
+npm run posthog:quality    # → nx run posthog-tools:quality:live
 npm run posthog:generate-types  # → regenerate types/posthog-api.gen.ts
 ```
 
@@ -39,6 +41,7 @@ Direct Nx invocations work too:
 nx run posthog-tools:sync:plan
 nx run posthog-tools:sync:apply
 nx run posthog-tools:sync:apply --args="--delete-orphans"
+nx run posthog-tools:quality:live -- --days 7 --limit-per-event 25
 nx run posthog-tools:test
 nx run posthog-tools:lint
 ```
@@ -90,6 +93,13 @@ Env vars (see `.env.example` at repo root):
 
 Event names must match [`docs/gtm/taxonomy.md`](../../docs/gtm/taxonomy.md). The `taxonomy.spec.ts` test enforces this on every CI run.
 
+## Data quality checks
+
+`telemetry-contract.ts` is the machine-readable event/property contract used by tests and live checks.
+
+- `taxonomy.spec.ts` and `telemetry-contract.spec.ts` guard committed dashboard JSON against undocumented events, unsupported breakdowns, unsupported filters, runtime dashboard coverage drift, and forbidden sensitive runtime fields.
+- `npm run posthog:quality -- --days 7 --limit-per-event 25` samples recent live PostHog events and validates observed payloads against the same contract. It exits non-zero for missing required properties or forbidden sensitive properties, and prints warnings for non-contract fields.
+
 ## Sync semantics
 
 - **`--plan`** — diff against PostHog, no writes. Outputs `[create] [update] [orphan]` per artifact. CI runs this on every PR that affects `posthog-tools`.

tools/posthog/live-quality.spec.ts

@@ -0,0 +1,169 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  analyzeTelemetryEvents,
+  fetchRecentContractEvents,
+  formatLiveQualityReport,
+  hasBlockingFindings,
+  type LiveTelemetryEvent,
+} from './live-quality.js';
+import { TELEMETRY_EVENT_CONTRACT } from './telemetry-contract.js';
+
+test('analyzeTelemetryEvents flags missing required and forbidden properties', () => {
+  const findings = analyzeTelemetryEvents([
+    {
+      event: 'ngaf:runtime_request_created',
+      timestamp: '2026-05-17T00:00:00Z',
+      properties: { messages: [{ content: 'hello' }] },
+    },
+  ]);
+
+  assert.deepEqual(
+    findings.map((finding) => ({
+      severity: finding.severity,
+      event: finding.event,
+      property: finding.property,
+      kind: finding.kind,
+    })),
+    [
+      {
+        severity: 'error',
+        event: 'ngaf:runtime_request_created',
+        property: 'transport',
+        kind: 'missing_required_property',
+      },
+      {
+        severity: 'error',
+        event: 'ngaf:runtime_request_created',
+        property: 'messages',
+        kind: 'forbidden_property',
+      },
+    ],
+  );
+  assert.equal(hasBlockingFindings(findings), true);
+});
+
+test('analyzeTelemetryEvents warns on non-contract properties but ignores PostHog metadata', () => {
+  const findings = analyzeTelemetryEvents([
+    {
+      event: 'ngaf:stream_started',
+      timestamp: '2026-05-17T00:00:00Z',
+      properties: {
+        transport: 'langgraph',
+        accidental_extra: true,
+        $current_url: 'https://example.test',
+        token: 'phc_x',
+      },
+    },
+  ]);
+
+  assert.deepEqual(
+    findings.map((finding) => ({
+      severity: finding.severity,
+      property: finding.property,
+      kind: finding.kind,
+    })),
+    [
+      {
+        severity: 'warning',
+        property: 'accidental_extra',
+        kind: 'unexpected_property',
+      },
+    ],
+  );
+  assert.equal(hasBlockingFindings(findings), false);
+});
+
+test('formatLiveQualityReport summarizes clean coverage and warnings', () => {
+  const events: LiveTelemetryEvent[] = [
+    {
+      event: 'ngaf:stream_started',
+      timestamp: '2026-05-17T00:00:00Z',
+      properties: { transport: 'langgraph', unexpected: true },
+    },
+  ];
+  const findings = analyzeTelemetryEvents(events);
+
+  const report = formatLiveQualityReport({
+    days: 1,
+    events,
+    findings,
+    checkedEvents: ['ngaf:stream_started', 'ngaf:stream_ended'],
+  });
+
+  assert.match(report, /Live telemetry quality — last 1 day/);
+  assert.match(report, /\| ngaf:stream_started \| 1 \|/);
+  assert.match(report, /\| ngaf:stream_ended \| 0 \|/);
+  assert.match(report, /Warnings/);
+  assert.match(report, /unexpected/);
+});
+
+test('fetchRecentContractEvents requests each contract event with bounded limits', async () => {
+  const calls: unknown[] = [];
+  const client = {
+    GET: async (path: string, options: unknown) => {
+      calls.push({ path, options });
+      return {
+        data: {
+          results: [
+            {
+              event: 'ngaf:stream_started',
+              timestamp: '2026-05-17T00:00:00Z',
+              properties: { transport: 'langgraph' },
+            },
+          ],
+        },
+      };
+    },
+  };
+
+  const events = await fetchRecentContractEvents({
+    client,
+    eventNames: ['ngaf:stream_started', 'ngaf:stream_ended'],
+    after: '2026-05-16T00:00:00.000Z',
+    limitPerEvent: 25,
+  });
+
+  assert.equal(events.length, 2);
+  assert.equal(calls.length, 2);
+  assert.deepEqual(calls, [
+    {
+      path: '/events/',
+      options: {
+        params: {
+          query: {
+            after: '2026-05-16T00:00:00.000Z',
+            event: 'ngaf:stream_started',
+            format: 'json',
+            limit: 25,
+          },
+        },
+      },
+    },
+    {
+      path: '/events/',
+      options: {
+        params: {
+          query: {
+            after: '2026-05-16T00:00:00.000Z',
+            event: 'ngaf:stream_ended',
+            format: 'json',
+            limit: 25,
+          },
+        },
+      },
+    },
+  ]);
+});
+
+test('every contracted event can be analyzed without a bespoke case', () => {
+  const events = Object.keys(TELEMETRY_EVENT_CONTRACT).map((event) => ({
+    event,
+    timestamp: '2026-05-17T00:00:00Z',
+    properties: Object.fromEntries(
+      TELEMETRY_EVENT_CONTRACT[event].requiredProperties.map((property) => [property, 'x']),
+    ),
+  }));
+
+  assert.equal(analyzeTelemetryEvents(events).some((finding) => finding.severity === 'error'), false);
+});