diff --git a/tools/posthog/sync.ts b/tools/posthog/sync.ts index 3e4504642..5901cd343 100644 --- a/tools/posthog/sync.ts +++ b/tools/posthog/sync.ts @@ -168,15 +168,6 @@ function resolveTiles( }); } -// PostHog rejects/ignores a `tiles` field on dashboard create/update. -// We keep tiles in the local JSON as the source of truth, but strip them -// from the body sent to PostHog. The wiring pass populates each insight's -// `dashboards` field instead. -function stripTiles(local: any): any { - const { tiles: _tiles, ...rest } = local; - return rest; -} - // Map our local InsightLocal shape (flat fields per zod schema) to PostHog's // modern query schema (HogQL-style InsightVizNode). The legacy `filters` shape // is rejected by modern PostHog projects. diff --git a/tools/posthog/telemetry-contract.spec.ts b/tools/posthog/telemetry-contract.spec.ts new file mode 100644 index 000000000..b40113100 --- /dev/null +++ b/tools/posthog/telemetry-contract.spec.ts @@ -0,0 +1,151 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFile, readdir } from 'node:fs/promises'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + NGAF_RUNTIME_EVENTS, + TELEMETRY_EVENT_CONTRACT, + TELEMETRY_FORBIDDEN_PROPERTIES, +} from './telemetry-contract.js'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = join(HERE, '..', '..'); +const INSIGHTS_DIR = join(HERE, 'insights'); + +async function readJson(path: string): Promise { + return JSON.parse(await readFile(path, 'utf8')) as T; +} + +async function insightFiles(): Promise { + return (await readdir(INSIGHTS_DIR)).filter((file) => file.endsWith('.json')).sort(); +} + +test('every insight event is registered in the telemetry contract', async () => { + const referenced = new Set(); + for (const file of await insightFiles()) { + const insight = await readJson<{ events?: Array<{ event?: string }>; steps?: Array<{ event?: string }> }>( + join(INSIGHTS_DIR, file), + ); + for (const item of [...(insight.events ?? []), ...(insight.steps ?? [])]) { + if (item.event) referenced.add(item.event); + } + } + + const unregistered = [...referenced].filter((event) => !(event in TELEMETRY_EVENT_CONTRACT)).sort(); + assert.deepEqual( + unregistered, + [], + `Insights reference events missing from TELEMETRY_EVENT_CONTRACT:\n${unregistered.join('\n')}`, + ); +}); + +test('insight breakdown properties are allowed by the event contract', async () => { + const violations: string[] = []; + for (const file of await insightFiles()) { + const insight = await readJson<{ slug: string; breakdown?: string; events?: Array<{ event?: string }> }>( + join(INSIGHTS_DIR, file), + ); + if (!insight.breakdown) continue; + for (const item of insight.events ?? []) { + const event = item.event; + if (!event) continue; + const allowed = TELEMETRY_EVENT_CONTRACT[event]?.allowedBreakdowns ?? []; + if (!allowed.includes(insight.breakdown)) { + violations.push(`${insight.slug}: ${event} breaks down by ${insight.breakdown}`); + } + } + } + + assert.deepEqual( + violations, + [], + `Insight breakdowns are not allowed by the telemetry contract:\n${violations.join('\n')}`, + ); +}); + +test('insight property filters are allowed by the event contract', async () => { + const violations: string[] = []; + for (const file of await insightFiles()) { + const insight = await readJson<{ + slug: string; + events?: Array<{ event?: string; properties?: Array<{ key?: string }> }>; + }>(join(INSIGHTS_DIR, file)); + + for (const item of insight.events ?? []) { + const event = item.event; + if (!event) continue; + const allowed = TELEMETRY_EVENT_CONTRACT[event]?.allowedProperties ?? []; + for (const property of item.properties ?? []) { + if (property.key && !allowed.includes(property.key)) { + violations.push(`${insight.slug}: ${event} filters by ${property.key}`); + } + } + } + } + + assert.deepEqual( + violations, + [], + `Insight property filters are not allowed by the telemetry contract:\n${violations.join('\n')}`, + ); +}); + +test('runtime dashboard covers every runtime event exactly once', async () => { + const dashboard = await readJson<{ tiles: Array<{ insight: string }> }>( + join(HERE, 'dashboards', 'runtime-telemetry.json'), + ); + + const coveredEventCounts = new Map(); + for (const tile of dashboard.tiles) { + const insight = await readJson<{ events?: Array<{ event?: string }> }>( + join(INSIGHTS_DIR, `${tile.insight}.json`), + ); + for (const item of insight.events ?? []) { + if (item.event) { + coveredEventCounts.set(item.event, (coveredEventCounts.get(item.event) ?? 0) + 1); + } + } + } + + const actualCoverage = [...coveredEventCounts.entries()] + .sort(([leftEvent], [rightEvent]) => leftEvent.localeCompare(rightEvent)); + const expectedCoverage: Array<[string, number]> = NGAF_RUNTIME_EVENTS + .map((event): [string, number] => [event, 1]) + .sort(([leftEvent], [rightEvent]) => leftEvent.localeCompare(rightEvent)); + + assert.deepEqual(actualCoverage, expectedCoverage); +}); + +test('public AgentRuntimeTelemetryEvent union matches the runtime event contract', async () => { + const runtimeTelemetrySource = await readFile( + join(REPO_ROOT, 'libs', 'chat', 'src', 'lib', 'agent', 'runtime-telemetry.ts'), + 'utf8', + ); + const match = runtimeTelemetrySource.match(/export type AgentRuntimeTelemetryEvent =([\s\S]*?);/); + assert(match, 'AgentRuntimeTelemetryEvent type not found'); + + const exportedEvents = [...match[1].matchAll(/'([^']+)'/g)].map((m) => m[1]).sort(); + assert.deepEqual(exportedEvents, [...NGAF_RUNTIME_EVENTS].sort()); +}); + +test('sensitive runtime fields are forbidden and never allowed by any event contract', () => { + assert.deepEqual( + [...TELEMETRY_FORBIDDEN_PROPERTIES].sort(), + ['apiUrl', 'assistantId', 'error', 'errorMessage', 'messages', 'prompt', 'query', 'threadId'].sort(), + ); + + const forbiddenProperties = new Set(TELEMETRY_FORBIDDEN_PROPERTIES); + const violations = Object.entries(TELEMETRY_EVENT_CONTRACT) + .flatMap(([event, contract]) => + contract.allowedProperties + .filter((property) => forbiddenProperties.has(property)) + .map((property) => `${event}: ${property}`), + ); + + assert.deepEqual( + violations, + [], + `Sensitive properties must not be allowed by telemetry contracts:\n${violations.join('\n')}`, + ); +}); diff --git a/tools/posthog/telemetry-contract.ts b/tools/posthog/telemetry-contract.ts new file mode 100644 index 000000000..75d4b13fd --- /dev/null +++ b/tools/posthog/telemetry-contract.ts @@ -0,0 +1,167 @@ +export const TELEMETRY_FORBIDDEN_PROPERTIES = [ + 'apiUrl', + 'assistantId', + 'error', + 'errorMessage', + 'messages', + 'prompt', + 'query', + 'threadId', +] as const; + +export const NGAF_RUNTIME_EVENTS = [ + 'ngaf:runtime_instance_created', + 'ngaf:runtime_request_created', + 'ngaf:stream_started', + 'ngaf:stream_ended', + 'ngaf:stream_errored', +] as const; + +type TelemetryEventContract = { + requiredProperties: readonly string[]; + allowedProperties: readonly string[]; + allowedBreakdowns: readonly string[]; +}; + +const installProperties = [ + 'arch', + 'global_install', + 'node', + 'node_version', + 'os', + 'package_manager', + 'package_manager_arch', + 'package_manager_node_version', + 'package_manager_os', + 'package_manager_version', + 'package_manager_workspaces', + 'pkg', + 'sample_weight', + 'version', +] as const; + +const runtimeProperties = [ + 'durationMs', + 'errorClass', + 'model', + 'provider', + 'requestType', + 'sample_weight', + 'surface', + 'transport', +] as const; + +const ctaProperties = [ + 'cta_id', + 'cta_text', + 'destination_url', + 'source_page', + 'source_section', + 'surface', + 'track', +] as const; + +export const TELEMETRY_EVENT_CONTRACT: Record = { + '$pageview': { + requiredProperties: [], + allowedProperties: ['$pathname'], + allowedBreakdowns: ['$pathname'], + }, + 'cockpit:activation_complete': { + requiredProperties: ['capability'], + allowedProperties: ['capability'], + allowedBreakdowns: ['capability'], + }, + 'cockpit:chat_first_message': { + requiredProperties: ['capability'], + allowedProperties: ['capability'], + allowedBreakdowns: ['capability'], + }, + 'cockpit:code_copied': { + requiredProperties: ['capability'], + allowedProperties: ['capability'], + allowedBreakdowns: ['capability'], + }, + 'cockpit:generative_component_rendered': { + requiredProperties: ['capability'], + allowedProperties: ['capability'], + allowedBreakdowns: ['capability'], + }, + 'cockpit:interrupt_handled': { + requiredProperties: ['capability'], + allowedProperties: ['capability'], + allowedBreakdowns: ['capability'], + }, + 'cockpit:mode_switched': { + requiredProperties: ['capability'], + allowedProperties: ['capability'], + allowedBreakdowns: ['capability'], + }, + 'cockpit:recipe_opened': { + requiredProperties: ['capability'], + allowedProperties: ['capability'], + allowedBreakdowns: ['capability'], + }, + 'cockpit:thread_persisted': { + requiredProperties: ['capability'], + allowedProperties: ['capability'], + allowedBreakdowns: ['capability'], + }, + 'cockpit:transport_connected': { + requiredProperties: ['capability'], + allowedProperties: ['capability'], + allowedBreakdowns: ['capability'], + }, + 'marketing:cta_click': { + requiredProperties: ['cta_id'], + allowedProperties: ctaProperties, + allowedBreakdowns: ['cta_id', 'source_page', 'source_section', 'surface', 'track'], + }, + 'ngaf:browser_chat_init': { + requiredProperties: ['surface'], + allowedProperties: runtimeProperties, + allowedBreakdowns: ['surface'], + }, + 'ngaf:browser_provided': { + requiredProperties: [], + allowedProperties: runtimeProperties, + allowedBreakdowns: ['surface'], + }, + 'ngaf:postinstall': { + requiredProperties: ['pkg', 'version'], + allowedProperties: installProperties, + allowedBreakdowns: [ + 'global_install', + 'os', + 'package_manager', + 'package_manager_os', + 'package_manager_workspaces', + 'pkg', + ], + }, + 'ngaf:runtime_instance_created': { + requiredProperties: ['transport'], + allowedProperties: runtimeProperties, + allowedBreakdowns: ['model', 'provider', 'requestType', 'surface', 'transport'], + }, + 'ngaf:runtime_request_created': { + requiredProperties: ['transport'], + allowedProperties: runtimeProperties, + allowedBreakdowns: ['model', 'provider', 'requestType', 'surface', 'transport'], + }, + 'ngaf:stream_started': { + requiredProperties: ['transport'], + allowedProperties: runtimeProperties, + allowedBreakdowns: ['model', 'provider', 'requestType', 'surface', 'transport'], + }, + 'ngaf:stream_ended': { + requiredProperties: ['transport'], + allowedProperties: runtimeProperties, + allowedBreakdowns: ['model', 'provider', 'requestType', 'surface', 'transport'], + }, + 'ngaf:stream_errored': { + requiredProperties: ['transport'], + allowedProperties: runtimeProperties, + allowedBreakdowns: ['errorClass', 'model', 'provider', 'requestType', 'surface', 'transport'], + }, +};