Add telemetry data quality guardrails

tools/posthog/sync.ts

@@ -168,15 +168,6 @@ function resolveTiles(
   });
 }
 
-// PostHog rejects/ignores a `tiles` field on dashboard create/update.
-// We keep tiles in the local JSON as the source of truth, but strip them
-// from the body sent to PostHog. The wiring pass populates each insight's
-// `dashboards` field instead.
-function stripTiles(local: any): any {
-  const { tiles: _tiles, ...rest } = local;
-  return rest;
-}
-
 // Map our local InsightLocal shape (flat fields per zod schema) to PostHog's
 // modern query schema (HogQL-style InsightVizNode). The legacy `filters` shape
 // is rejected by modern PostHog projects.

tools/posthog/telemetry-contract.spec.ts

@@ -0,0 +1,151 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { readFile, readdir } from 'node:fs/promises';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import {
+  NGAF_RUNTIME_EVENTS,
+  TELEMETRY_EVENT_CONTRACT,
+  TELEMETRY_FORBIDDEN_PROPERTIES,
+} from './telemetry-contract.js';
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = join(HERE, '..', '..');
+const INSIGHTS_DIR = join(HERE, 'insights');
+
+async function readJson<T>(path: string): Promise<T> {
+  return JSON.parse(await readFile(path, 'utf8')) as T;
+}
+
+async function insightFiles(): Promise<string[]> {
+  return (await readdir(INSIGHTS_DIR)).filter((file) => file.endsWith('.json')).sort();
+}
+
+test('every insight event is registered in the telemetry contract', async () => {
+  const referenced = new Set<string>();
+  for (const file of await insightFiles()) {
+    const insight = await readJson<{ events?: Array<{ event?: string }>; steps?: Array<{ event?: string }> }>(
+      join(INSIGHTS_DIR, file),
+    );
+    for (const item of [...(insight.events ?? []), ...(insight.steps ?? [])]) {
+      if (item.event) referenced.add(item.event);
+    }
+  }
+
+  const unregistered = [...referenced].filter((event) => !(event in TELEMETRY_EVENT_CONTRACT)).sort();
+  assert.deepEqual(
+    unregistered,
+    [],
+    `Insights reference events missing from TELEMETRY_EVENT_CONTRACT:\n${unregistered.join('\n')}`,
+  );
+});
+
+test('insight breakdown properties are allowed by the event contract', async () => {
+  const violations: string[] = [];
+  for (const file of await insightFiles()) {
+    const insight = await readJson<{ slug: string; breakdown?: string; events?: Array<{ event?: string }> }>(
+      join(INSIGHTS_DIR, file),
+    );
+    if (!insight.breakdown) continue;
+    for (const item of insight.events ?? []) {
+      const event = item.event;
+      if (!event) continue;
+      const allowed = TELEMETRY_EVENT_CONTRACT[event]?.allowedBreakdowns ?? [];
+      if (!allowed.includes(insight.breakdown)) {
+        violations.push(`${insight.slug}: ${event} breaks down by ${insight.breakdown}`);
+      }
+    }
+  }
+
+  assert.deepEqual(
+    violations,
+    [],
+    `Insight breakdowns are not allowed by the telemetry contract:\n${violations.join('\n')}`,
+  );
+});
+
+test('insight property filters are allowed by the event contract', async () => {
+  const violations: string[] = [];
+  for (const file of await insightFiles()) {
+    const insight = await readJson<{
+      slug: string;
+      events?: Array<{ event?: string; properties?: Array<{ key?: string }> }>;
+    }>(join(INSIGHTS_DIR, file));
+
+    for (const item of insight.events ?? []) {
+      const event = item.event;
+      if (!event) continue;
+      const allowed = TELEMETRY_EVENT_CONTRACT[event]?.allowedProperties ?? [];
+      for (const property of item.properties ?? []) {
+        if (property.key && !allowed.includes(property.key)) {
+          violations.push(`${insight.slug}: ${event} filters by ${property.key}`);
+        }
+      }
+    }
+  }
+
+  assert.deepEqual(
+    violations,
+    [],
+    `Insight property filters are not allowed by the telemetry contract:\n${violations.join('\n')}`,
+  );
+});
+
+test('runtime dashboard covers every runtime event exactly once', async () => {
+  const dashboard = await readJson<{ tiles: Array<{ insight: string }> }>(
+    join(HERE, 'dashboards', 'runtime-telemetry.json'),
+  );
+
+  const coveredEventCounts = new Map<string, number>();
+  for (const tile of dashboard.tiles) {
+    const insight = await readJson<{ events?: Array<{ event?: string }> }>(
+      join(INSIGHTS_DIR, `${tile.insight}.json`),
+    );
+    for (const item of insight.events ?? []) {
+      if (item.event) {
+        coveredEventCounts.set(item.event, (coveredEventCounts.get(item.event) ?? 0) + 1);
+      }
+    }
+  }
+
+  const actualCoverage = [...coveredEventCounts.entries()]
+    .sort(([leftEvent], [rightEvent]) => leftEvent.localeCompare(rightEvent));
+  const expectedCoverage: Array<[string, number]> = NGAF_RUNTIME_EVENTS
+    .map((event): [string, number] => [event, 1])
+    .sort(([leftEvent], [rightEvent]) => leftEvent.localeCompare(rightEvent));
+
+  assert.deepEqual(actualCoverage, expectedCoverage);
+});
+
+test('public AgentRuntimeTelemetryEvent union matches the runtime event contract', async () => {
+  const runtimeTelemetrySource = await readFile(
+    join(REPO_ROOT, 'libs', 'chat', 'src', 'lib', 'agent', 'runtime-telemetry.ts'),
+    'utf8',
+  );
+  const match = runtimeTelemetrySource.match(/export type AgentRuntimeTelemetryEvent =([\s\S]*?);/);
+  assert(match, 'AgentRuntimeTelemetryEvent type not found');
+
+  const exportedEvents = [...match[1].matchAll(/'([^']+)'/g)].map((m) => m[1]).sort();
+  assert.deepEqual(exportedEvents, [...NGAF_RUNTIME_EVENTS].sort());
+});
+
+test('sensitive runtime fields are forbidden and never allowed by any event contract', () => {
+  assert.deepEqual(
+    [...TELEMETRY_FORBIDDEN_PROPERTIES].sort(),
+    ['apiUrl', 'assistantId', 'error', 'errorMessage', 'messages', 'prompt', 'query', 'threadId'].sort(),
+  );
+
+  const forbiddenProperties = new Set<string>(TELEMETRY_FORBIDDEN_PROPERTIES);
+  const violations = Object.entries(TELEMETRY_EVENT_CONTRACT)
+    .flatMap(([event, contract]) =>
+      contract.allowedProperties
+        .filter((property) => forbiddenProperties.has(property))
+        .map((property) => `${event}: ${property}`),
+    );
+
+  assert.deepEqual(
+    violations,
+    [],
+    `Sensitive properties must not be allowed by telemetry contracts:\n${violations.join('\n')}`,
+  );
+});

tools/posthog/telemetry-contract.ts

@@ -0,0 +1,167 @@
+export const TELEMETRY_FORBIDDEN_PROPERTIES = [
+  'apiUrl',
+  'assistantId',
+  'error',
+  'errorMessage',
+  'messages',
+  'prompt',
+  'query',
+  'threadId',
+] as const;
+
+export const NGAF_RUNTIME_EVENTS = [
+  'ngaf:runtime_instance_created',
+  'ngaf:runtime_request_created',
+  'ngaf:stream_started',
+  'ngaf:stream_ended',
+  'ngaf:stream_errored',
+] as const;
+
+type TelemetryEventContract = {
+  requiredProperties: readonly string[];
+  allowedProperties: readonly string[];
+  allowedBreakdowns: readonly string[];
+};
+
+const installProperties = [
+  'arch',
+  'global_install',
+  'node',
+  'node_version',
+  'os',
+  'package_manager',
+  'package_manager_arch',
+  'package_manager_node_version',
+  'package_manager_os',
+  'package_manager_version',
+  'package_manager_workspaces',
+  'pkg',
+  'sample_weight',
+  'version',
+] as const;
+
+const runtimeProperties = [
+  'durationMs',
+  'errorClass',
+  'model',
+  'provider',
+  'requestType',
+  'sample_weight',
+  'surface',
+  'transport',
+] as const;
+
+const ctaProperties = [
+  'cta_id',
+  'cta_text',
+  'destination_url',
+  'source_page',
+  'source_section',
+  'surface',
+  'track',
+] as const;
+
+export const TELEMETRY_EVENT_CONTRACT: Record<string, TelemetryEventContract> = {
+  '$pageview': {
+    requiredProperties: [],
+    allowedProperties: ['$pathname'],
+    allowedBreakdowns: ['$pathname'],
+  },
+  'cockpit:activation_complete': {
+    requiredProperties: ['capability'],
+    allowedProperties: ['capability'],
+    allowedBreakdowns: ['capability'],
+  },
+  'cockpit:chat_first_message': {
+    requiredProperties: ['capability'],
+    allowedProperties: ['capability'],
+    allowedBreakdowns: ['capability'],
+  },
+  'cockpit:code_copied': {
+    requiredProperties: ['capability'],
+    allowedProperties: ['capability'],
+    allowedBreakdowns: ['capability'],
+  },
+  'cockpit:generative_component_rendered': {
+    requiredProperties: ['capability'],
+    allowedProperties: ['capability'],
+    allowedBreakdowns: ['capability'],
+  },
+  'cockpit:interrupt_handled': {
+    requiredProperties: ['capability'],
+    allowedProperties: ['capability'],
+    allowedBreakdowns: ['capability'],
+  },
+  'cockpit:mode_switched': {
+    requiredProperties: ['capability'],
+    allowedProperties: ['capability'],
+    allowedBreakdowns: ['capability'],
+  },
+  'cockpit:recipe_opened': {
+    requiredProperties: ['capability'],
+    allowedProperties: ['capability'],
+    allowedBreakdowns: ['capability'],
+  },
+  'cockpit:thread_persisted': {
+    requiredProperties: ['capability'],
+    allowedProperties: ['capability'],
+    allowedBreakdowns: ['capability'],
+  },
+  'cockpit:transport_connected': {
+    requiredProperties: ['capability'],
+    allowedProperties: ['capability'],
+    allowedBreakdowns: ['capability'],
+  },
+  'marketing:cta_click': {
+    requiredProperties: ['cta_id'],
+    allowedProperties: ctaProperties,
+    allowedBreakdowns: ['cta_id', 'source_page', 'source_section', 'surface', 'track'],
+  },
+  'ngaf:browser_chat_init': {
+    requiredProperties: ['surface'],
+    allowedProperties: runtimeProperties,
+    allowedBreakdowns: ['surface'],
+  },
+  'ngaf:browser_provided': {
+    requiredProperties: [],
+    allowedProperties: runtimeProperties,
+    allowedBreakdowns: ['surface'],
+  },
+  'ngaf:postinstall': {
+    requiredProperties: ['pkg', 'version'],
+    allowedProperties: installProperties,
+    allowedBreakdowns: [
+      'global_install',
+      'os',
+      'package_manager',
+      'package_manager_os',
+      'package_manager_workspaces',
+      'pkg',
+    ],
+  },
+  'ngaf:runtime_instance_created': {
+    requiredProperties: ['transport'],
+    allowedProperties: runtimeProperties,
+    allowedBreakdowns: ['model', 'provider', 'requestType', 'surface', 'transport'],
+  },
+  'ngaf:runtime_request_created': {
+    requiredProperties: ['transport'],
+    allowedProperties: runtimeProperties,
+    allowedBreakdowns: ['model', 'provider', 'requestType', 'surface', 'transport'],
+  },
+  'ngaf:stream_started': {
+    requiredProperties: ['transport'],
+    allowedProperties: runtimeProperties,
+    allowedBreakdowns: ['model', 'provider', 'requestType', 'surface', 'transport'],
+  },
+  'ngaf:stream_ended': {
+    requiredProperties: ['transport'],
+    allowedProperties: runtimeProperties,
+    allowedBreakdowns: ['model', 'provider', 'requestType', 'surface', 'transport'],
+  },
+  'ngaf:stream_errored': {
+    requiredProperties: ['transport'],
+    allowedProperties: runtimeProperties,
+    allowedBreakdowns: ['errorClass', 'model', 'provider', 'requestType', 'surface', 'transport'],
+  },
+};