diff --git a/package.json b/package.json index 65e53c295..c818b0a71 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "posthog:sync": "nx run posthog-tools:sync:plan", "posthog:apply": "nx run posthog-tools:sync:apply", "posthog:report": "nx run posthog-tools:report", + "posthog:quality": "nx run posthog-tools:quality:live", "posthog:generate-types": "nx run posthog-tools:generate-types", "telemetry:install-smoke": "node libs/telemetry/scripts/smoke-install-telemetry.mjs dist/libs/chat dist/libs/langgraph dist/libs/ag-ui dist/libs/render dist/libs/a2ui dist/libs/licensing dist/libs/telemetry" }, diff --git a/tools/posthog/README.md b/tools/posthog/README.md index e554ee0f6..d79da2087 100644 --- a/tools/posthog/README.md +++ b/tools/posthog/README.md @@ -14,6 +14,7 @@ tools/posthog/ ├── schema.ts # zod schemas for local JSON ├── sync.ts # CLI: plan / apply / writeback ├── report.ts # CLI: pull insights → markdown +├── live-quality.ts # CLI: sample recent events and validate payload quality ├── *.spec.ts # tests ├── types/posthog-api.gen.ts # generated from PostHog OpenAPI spec ├── scripts/generate-types.ts # regenerate the above @@ -30,6 +31,7 @@ All commands wrap `nx run posthog-tools:*`. Root-package aliases: npm run posthog:sync # → nx run posthog-tools:sync:plan npm run posthog:apply # → nx run posthog-tools:sync:apply npm run posthog:report # → nx run posthog-tools:report +npm run posthog:quality # → nx run posthog-tools:quality:live npm run posthog:generate-types # → regenerate types/posthog-api.gen.ts ``` @@ -39,6 +41,7 @@ Direct Nx invocations work too: nx run posthog-tools:sync:plan nx run posthog-tools:sync:apply nx run posthog-tools:sync:apply --args="--delete-orphans" +nx run posthog-tools:quality:live -- --days 7 --limit-per-event 25 nx run posthog-tools:test nx run posthog-tools:lint ``` @@ -90,6 +93,13 @@ Env vars (see `.env.example` at repo root): Event names must match [`docs/gtm/taxonomy.md`](../../docs/gtm/taxonomy.md). The `taxonomy.spec.ts` test enforces this on every CI run. +## Data quality checks + +`telemetry-contract.ts` is the machine-readable event/property contract used by tests and live checks. + +- `taxonomy.spec.ts` and `telemetry-contract.spec.ts` guard committed dashboard JSON against undocumented events, unsupported breakdowns, unsupported filters, runtime dashboard coverage drift, and forbidden sensitive runtime fields. +- `npm run posthog:quality -- --days 7 --limit-per-event 25` samples recent live PostHog events and validates observed payloads against the same contract. It exits non-zero for missing required properties or forbidden sensitive properties, and prints warnings for non-contract fields. + ## Sync semantics - **`--plan`** — diff against PostHog, no writes. Outputs `[create] [update] [orphan]` per artifact. CI runs this on every PR that affects `posthog-tools`. diff --git a/tools/posthog/live-quality.spec.ts b/tools/posthog/live-quality.spec.ts new file mode 100644 index 000000000..bf604a89a --- /dev/null +++ b/tools/posthog/live-quality.spec.ts @@ -0,0 +1,169 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + analyzeTelemetryEvents, + fetchRecentContractEvents, + formatLiveQualityReport, + hasBlockingFindings, + type LiveTelemetryEvent, +} from './live-quality.js'; +import { TELEMETRY_EVENT_CONTRACT } from './telemetry-contract.js'; + +test('analyzeTelemetryEvents flags missing required and forbidden properties', () => { + const findings = analyzeTelemetryEvents([ + { + event: 'ngaf:runtime_request_created', + timestamp: '2026-05-17T00:00:00Z', + properties: { messages: [{ content: 'hello' }] }, + }, + ]); + + assert.deepEqual( + findings.map((finding) => ({ + severity: finding.severity, + event: finding.event, + property: finding.property, + kind: finding.kind, + })), + [ + { + severity: 'error', + event: 'ngaf:runtime_request_created', + property: 'transport', + kind: 'missing_required_property', + }, + { + severity: 'error', + event: 'ngaf:runtime_request_created', + property: 'messages', + kind: 'forbidden_property', + }, + ], + ); + assert.equal(hasBlockingFindings(findings), true); +}); + +test('analyzeTelemetryEvents warns on non-contract properties but ignores PostHog metadata', () => { + const findings = analyzeTelemetryEvents([ + { + event: 'ngaf:stream_started', + timestamp: '2026-05-17T00:00:00Z', + properties: { + transport: 'langgraph', + accidental_extra: true, + $current_url: 'https://example.test', + token: 'phc_x', + }, + }, + ]); + + assert.deepEqual( + findings.map((finding) => ({ + severity: finding.severity, + property: finding.property, + kind: finding.kind, + })), + [ + { + severity: 'warning', + property: 'accidental_extra', + kind: 'unexpected_property', + }, + ], + ); + assert.equal(hasBlockingFindings(findings), false); +}); + +test('formatLiveQualityReport summarizes clean coverage and warnings', () => { + const events: LiveTelemetryEvent[] = [ + { + event: 'ngaf:stream_started', + timestamp: '2026-05-17T00:00:00Z', + properties: { transport: 'langgraph', unexpected: true }, + }, + ]; + const findings = analyzeTelemetryEvents(events); + + const report = formatLiveQualityReport({ + days: 1, + events, + findings, + checkedEvents: ['ngaf:stream_started', 'ngaf:stream_ended'], + }); + + assert.match(report, /Live telemetry quality — last 1 day/); + assert.match(report, /\| ngaf:stream_started \| 1 \|/); + assert.match(report, /\| ngaf:stream_ended \| 0 \|/); + assert.match(report, /Warnings/); + assert.match(report, /unexpected/); +}); + +test('fetchRecentContractEvents requests each contract event with bounded limits', async () => { + const calls: unknown[] = []; + const client = { + GET: async (path: string, options: unknown) => { + calls.push({ path, options }); + return { + data: { + results: [ + { + event: 'ngaf:stream_started', + timestamp: '2026-05-17T00:00:00Z', + properties: { transport: 'langgraph' }, + }, + ], + }, + }; + }, + }; + + const events = await fetchRecentContractEvents({ + client, + eventNames: ['ngaf:stream_started', 'ngaf:stream_ended'], + after: '2026-05-16T00:00:00.000Z', + limitPerEvent: 25, + }); + + assert.equal(events.length, 2); + assert.equal(calls.length, 2); + assert.deepEqual(calls, [ + { + path: '/events/', + options: { + params: { + query: { + after: '2026-05-16T00:00:00.000Z', + event: 'ngaf:stream_started', + format: 'json', + limit: 25, + }, + }, + }, + }, + { + path: '/events/', + options: { + params: { + query: { + after: '2026-05-16T00:00:00.000Z', + event: 'ngaf:stream_ended', + format: 'json', + limit: 25, + }, + }, + }, + }, + ]); +}); + +test('every contracted event can be analyzed without a bespoke case', () => { + const events = Object.keys(TELEMETRY_EVENT_CONTRACT).map((event) => ({ + event, + timestamp: '2026-05-17T00:00:00Z', + properties: Object.fromEntries( + TELEMETRY_EVENT_CONTRACT[event].requiredProperties.map((property) => [property, 'x']), + ), + })); + + assert.equal(analyzeTelemetryEvents(events).some((finding) => finding.severity === 'error'), false); +}); diff --git a/tools/posthog/live-quality.ts b/tools/posthog/live-quality.ts new file mode 100644 index 000000000..8917ad06e --- /dev/null +++ b/tools/posthog/live-quality.ts @@ -0,0 +1,240 @@ +import { ph } from './client.js'; +import { + TELEMETRY_EVENT_CONTRACT, + TELEMETRY_FORBIDDEN_PROPERTIES, +} from './telemetry-contract.js'; + +export interface LiveTelemetryEvent { + event: string; + timestamp: string; + properties: Record; +} + +export type LiveQualityFindingKind = + | 'missing_required_property' + | 'forbidden_property' + | 'unexpected_property'; + +export interface LiveQualityFinding { + severity: 'error' | 'warning'; + kind: LiveQualityFindingKind; + event: string; + property: string; + timestamp: string; + message: string; +} + +export interface FetchRecentContractEventsOptions { + client: LiveTelemetryEventsClient; + eventNames: readonly string[]; + after: string; + limitPerEvent: number; +} + +export interface LiveTelemetryEventsClient { + GET(path: string, options: unknown): Promise<{ data?: { results?: LiveTelemetryEvent[] }; error?: unknown }>; +} + +const INTERNAL_PROPERTY_NAMES = new Set([ + 'distinct_id', + 'token', + 'uuid', +]); + +function isMissing(value: unknown): boolean { + return value === undefined || value === null || value === ''; +} + +function isInternalProperty(property: string): boolean { + return property.startsWith('$') || INTERNAL_PROPERTY_NAMES.has(property); +} + +export function analyzeTelemetryEvents( + events: readonly LiveTelemetryEvent[], + contract = TELEMETRY_EVENT_CONTRACT, +): LiveQualityFinding[] { + const forbiddenProperties = new Set(TELEMETRY_FORBIDDEN_PROPERTIES); + const findings: LiveQualityFinding[] = []; + + for (const item of events) { + const eventContract = contract[item.event]; + if (!eventContract) continue; + + const allowedProperties = new Set(eventContract.allowedProperties); + for (const property of eventContract.requiredProperties) { + if (isMissing(item.properties[property])) { + findings.push({ + severity: 'error', + kind: 'missing_required_property', + event: item.event, + property, + timestamp: item.timestamp, + message: `${item.event} is missing required property ${property}`, + }); + } + } + + for (const property of Object.keys(item.properties)) { + if (forbiddenProperties.has(property)) { + findings.push({ + severity: 'error', + kind: 'forbidden_property', + event: item.event, + property, + timestamp: item.timestamp, + message: `${item.event} includes forbidden property ${property}`, + }); + continue; + } + + if (!isInternalProperty(property) && !allowedProperties.has(property)) { + findings.push({ + severity: 'warning', + kind: 'unexpected_property', + event: item.event, + property, + timestamp: item.timestamp, + message: `${item.event} includes non-contract property ${property}`, + }); + } + } + } + + return findings; +} + +export function hasBlockingFindings(findings: readonly LiveQualityFinding[]): boolean { + return findings.some((finding) => finding.severity === 'error'); +} + +export async function fetchRecentContractEvents({ + client, + eventNames, + after, + limitPerEvent, +}: FetchRecentContractEventsOptions): Promise { + const fetched: LiveTelemetryEvent[] = []; + for (const event of eventNames) { + const response = await client.GET('/events/', { + params: { + query: { + after, + event, + format: 'json', + limit: limitPerEvent, + }, + }, + }); + + if (response.error || response.data === undefined) { + throw new Error(`PostHog events query failed for ${event}: ${JSON.stringify(response.error)}`); + } + fetched.push(...(response.data.results ?? [])); + } + return fetched; +} + +export function formatLiveQualityReport({ + checkedEvents, + days, + events, + findings, +}: { + checkedEvents: readonly string[]; + days: number; + events: readonly LiveTelemetryEvent[]; + findings: readonly LiveQualityFinding[]; +}): string { + const lines: string[] = []; + lines.push(`Live telemetry quality — last ${days} ${days === 1 ? 'day' : 'days'}`); + lines.push(''); + lines.push('| Event | Sampled events |'); + lines.push('|-------|---------------:|'); + for (const event of checkedEvents) { + const count = events.filter((item) => item.event === event).length; + lines.push(`| ${event} | ${count} |`); + } + + const errors = findings.filter((finding) => finding.severity === 'error'); + const warnings = findings.filter((finding) => finding.severity === 'warning'); + lines.push(''); + lines.push(`Errors: ${errors.length}`); + lines.push(`Warnings: ${warnings.length}`); + + if (errors.length > 0) { + lines.push(''); + lines.push('## Errors'); + for (const finding of errors) { + lines.push(`- ${finding.message} (${finding.timestamp})`); + } + } + + if (warnings.length > 0) { + lines.push(''); + lines.push('## Warnings'); + for (const finding of warnings) { + lines.push(`- ${finding.message} (${finding.timestamp})`); + } + } + + return lines.join('\n'); +} + +function parseArgs(args: readonly string[]): { days: number; limitPerEvent: number } { + let days = 7; + let limitPerEvent = 100; + for (let i = 0; i < args.length; i += 1) { + const arg = args[i]; + if (arg === '--days') { + days = Number(args[i + 1]); + i += 1; + } else if (arg === '--limit-per-event') { + limitPerEvent = Number(args[i + 1]); + i += 1; + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + if (!Number.isInteger(days) || days <= 0) throw new Error('--days must be a positive integer'); + if (!Number.isInteger(limitPerEvent) || limitPerEvent <= 0) { + throw new Error('--limit-per-event must be a positive integer'); + } + return { days, limitPerEvent }; +} + +async function main(): Promise { + let options: { days: number; limitPerEvent: number }; + try { + options = parseArgs(process.argv.slice(2)); + } catch (err) { + console.error(err instanceof Error ? err.message : err); + console.error('Usage: tsx tools/posthog/live-quality.ts [--days N] [--limit-per-event N]'); + return 1; + } + + const checkedEvents = Object.keys(TELEMETRY_EVENT_CONTRACT).sort(); + const after = new Date(Date.now() - options.days * 24 * 60 * 60 * 1000).toISOString(); + try { + const events = await fetchRecentContractEvents({ + client: ph() as LiveTelemetryEventsClient, + eventNames: checkedEvents, + after, + limitPerEvent: options.limitPerEvent, + }); + const findings = analyzeTelemetryEvents(events); + console.log(formatLiveQualityReport({ + checkedEvents, + days: options.days, + events, + findings, + })); + return hasBlockingFindings(findings) ? 1 : 0; + } catch (err) { + console.error(`Live telemetry quality check failed: ${err instanceof Error ? err.message : err}`); + return 1; + } +} + +if (process.argv[1]?.endsWith('/live-quality.ts')) { + main().then((code) => process.exit(code)); +} diff --git a/tools/posthog/project.json b/tools/posthog/project.json index 3cfa2a8d0..4f471c8f7 100644 --- a/tools/posthog/project.json +++ b/tools/posthog/project.json @@ -37,6 +37,12 @@ "command": "npx tsx tools/posthog/report.ts" } }, + "quality:live": { + "executor": "nx:run-commands", + "options": { + "command": "npx tsx tools/posthog/live-quality.ts" + } + }, "generate-types": { "executor": "nx:run-commands", "options": { diff --git a/tools/posthog/telemetry-contract.ts b/tools/posthog/telemetry-contract.ts index 75d4b13fd..1f111563f 100644 --- a/tools/posthog/telemetry-contract.ts +++ b/tools/posthog/telemetry-contract.ts @@ -61,10 +61,20 @@ const ctaProperties = [ 'track', ] as const; +const cockpitShellProperties = [ + 'capability', + 'category', + 'file_path', + 'from_capability', + 'from_mode', + 'surface', + 'to_mode', +] as const; + export const TELEMETRY_EVENT_CONTRACT: Record = { '$pageview': { requiredProperties: [], - allowedProperties: ['$pathname'], + allowedProperties: ['$pathname', 'title'], allowedBreakdowns: ['$pathname'], }, 'cockpit:activation_complete': { @@ -79,8 +89,8 @@ export const TELEMETRY_EVENT_CONTRACT: Record = }, 'cockpit:code_copied': { requiredProperties: ['capability'], - allowedProperties: ['capability'], - allowedBreakdowns: ['capability'], + allowedProperties: cockpitShellProperties, + allowedBreakdowns: ['capability', 'surface'], }, 'cockpit:generative_component_rendered': { requiredProperties: ['capability'], @@ -94,13 +104,13 @@ export const TELEMETRY_EVENT_CONTRACT: Record = }, 'cockpit:mode_switched': { requiredProperties: ['capability'], - allowedProperties: ['capability'], - allowedBreakdowns: ['capability'], + allowedProperties: cockpitShellProperties, + allowedBreakdowns: ['capability', 'from_mode', 'to_mode'], }, 'cockpit:recipe_opened': { requiredProperties: ['capability'], - allowedProperties: ['capability'], - allowedBreakdowns: ['capability'], + allowedProperties: cockpitShellProperties, + allowedBreakdowns: ['capability', 'category', 'from_capability'], }, 'cockpit:thread_persisted': { requiredProperties: ['capability'],