diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4dcc1eca5..7f3fd6d42 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -46,6 +46,9 @@ jobs: - name: Patch install telemetry into publishable manifests run: node libs/telemetry/scripts/apply-install-telemetry.mjs dist/libs/chat dist/libs/langgraph dist/libs/ag-ui dist/libs/render dist/libs/a2ui dist/libs/licensing + - name: Verify install telemetry in publishable manifests + run: node libs/telemetry/scripts/verify-install-telemetry.mjs dist/libs/chat dist/libs/langgraph dist/libs/ag-ui dist/libs/render dist/libs/a2ui dist/libs/licensing dist/libs/telemetry + - name: Verify atomic release versions run: node scripts/verify-release-versions.mjs --tag "$RELEASE_TAG" env: diff --git a/apps/website/src/app/api/ingest/route.spec.ts b/apps/website/src/app/api/ingest/route.spec.ts new file mode 100644 index 000000000..950ebc872 --- /dev/null +++ b/apps/website/src/app/api/ingest/route.spec.ts @@ -0,0 +1,43 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const capture = vi.fn(); +const shutdown = vi.fn(); + +vi.mock('posthog-node', () => ({ + PostHog: vi.fn(function PostHog() { + return { capture, shutdown }; + }), +})); + +import { POST } from './route'; + +describe('/api/ingest', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env.NEXT_PUBLIC_POSTHOG_TOKEN = 'phc_server'; + delete process.env.NEXT_PUBLIC_POSTHOG_HOST; + }); + + it('accepts neutral browser telemetry payloads without requiring the public ingest key', async () => { + shutdown.mockResolvedValue(undefined); + const response = await POST(new Request('https://cacheplane.ai/api/ingest', { + method: 'POST', + body: JSON.stringify({ + event: 'ngaf:browser_chat_init', + distinctId: 'browser:test', + properties: { surface: 'canonical_demo' }, + }), + }) as never); + + expect(response.status).toBe(202); + expect(capture).toHaveBeenCalledWith({ + distinctId: 'browser:test', + event: 'ngaf:browser_chat_init', + properties: { + surface: 'canonical_demo', + $ip: null, + $process_person_profile: false, + }, + }); + }); +}); diff --git a/apps/website/src/app/api/ingest/route.ts b/apps/website/src/app/api/ingest/route.ts index dfc806c71..604b8c42d 100644 --- a/apps/website/src/app/api/ingest/route.ts +++ b/apps/website/src/app/api/ingest/route.ts @@ -32,7 +32,7 @@ function readPayload(value: unknown): { } | null { if (!isRecord(value)) return null; const payload = value as TelemetryIngestPayload; - if (payload.key !== PUBLIC_INGEST_KEY) return null; + if (payload.key !== undefined && payload.key !== PUBLIC_INGEST_KEY) return null; const distinctId = toSafeAnalyticsString(payload.distinctId, 200); const event = toSafeAnalyticsString(payload.event, 100); diff --git a/docs/gtm/taxonomy.md b/docs/gtm/taxonomy.md index ddebf4832..7e52a33aa 100644 --- a/docs/gtm/taxonomy.md +++ b/docs/gtm/taxonomy.md @@ -75,6 +75,7 @@ sessions by design. |--------------------------------------|--------------------------------------------|-----------------|--------------| | `ngaf:postinstall` | Dependency/global install of a published `@ngaf/*` package | Node (script) | **Opt-out** | | `ngaf:runtime_instance_created` | Runtime adapter init | Node / Browser | **Opt-out** on Node, **Opt-in** in Browser | +| `ngaf:runtime_request_created` | Runtime adapter request created | Node / Browser | **Opt-out** on Node, **Opt-in** in Browser | | `ngaf:stream_started` | Stream begins | Node / Browser | **Opt-out** on Node, **Opt-in** in Browser | | `ngaf:stream_ended` | Stream ends normally | Node / Browser | **Opt-out** on Node, **Opt-in** in Browser | | `ngaf:stream_errored` | Stream errors | Node / Browser | **Opt-out** on Node, **Opt-in** in Browser | @@ -102,6 +103,19 @@ Browser events never fire unless the consumer explicitly opts in. See `libs/tele | `global_install` | bool | Whether npm reports a global install. | | `sample_weight` | number | Inverse sample rate for weighted counts. | +### Runtime telemetry properties + +| Property | Type | Notes | +|---------------|--------|--------------------------------------------| +| `transport` | string | Runtime transport, e.g. `langgraph`, `ag-ui`, or `custom`. | +| `surface` | string | Adapter surface emitting the event. | +| `requestType` | string | Request shape, e.g. `submit`, `resubmit`, `regenerate`, `enqueue`, or `join`. | +| `provider` | string | Model provider when known. | +| `model` | string | Model name when known. | +| `durationMs` | number | Stream duration for end/error events. | +| `errorClass` | string | Error class only. Never send error messages. | +| `sample_weight` | number | Inverse sample rate for weighted counts. | + ## Shared properties | Property | Type | Notes | diff --git a/libs/ag-ui/src/lib/provide-ag-ui-agent.ts b/libs/ag-ui/src/lib/provide-ag-ui-agent.ts index d5dd019ce..ed4283584 100644 --- a/libs/ag-ui/src/lib/provide-ag-ui-agent.ts +++ b/libs/ag-ui/src/lib/provide-ag-ui-agent.ts @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT import { InjectionToken, inject, type Provider } from '@angular/core'; import { HttpAgent } from '@ag-ui/client'; -import type { Agent } from '@ngaf/chat'; +import type { Agent, AgentRuntimeTelemetrySink } from '@ngaf/chat'; import { toAgent } from './to-agent'; /** @@ -17,6 +17,8 @@ export interface AgUiAgentConfig { agentId?: string; threadId?: string; headers?: Record; + /** Optional app-owned telemetry sink. No telemetry is emitted unless this is provided. */ + telemetry?: AgentRuntimeTelemetrySink | false; } export const AG_UI_AGENT = new InjectionToken('AG_UI_AGENT'); @@ -38,7 +40,7 @@ export function provideAgUiAgent(config: AgUiAgentConfig): Provider[] { ...(config.threadId !== undefined ? { threadId: config.threadId } : {}), ...(config.headers !== undefined ? { headers: config.headers } : {}), }); - return toAgent(source); + return toAgent(source, { telemetry: config.telemetry }); }, }, ]; diff --git a/libs/ag-ui/src/lib/to-agent.spec.ts b/libs/ag-ui/src/lib/to-agent.spec.ts index f6598651d..a4f3cd54d 100644 --- a/libs/ag-ui/src/lib/to-agent.spec.ts +++ b/libs/ag-ui/src/lib/to-agent.spec.ts @@ -3,6 +3,7 @@ import { describe, it, expect, vi } from 'vitest'; import { Observable, Subject } from 'rxjs'; import type { AbstractAgent, BaseEvent } from '@ag-ui/client'; import type { RunAgentInput } from '@ag-ui/core'; +import type { AgentRuntimeTelemetryPayload } from '@ngaf/chat'; import { toAgent } from './to-agent'; /** @@ -113,6 +114,51 @@ describe('toAgent', () => { expect(stub.runAgent).toHaveBeenCalledOnce(); }); + it('emits opt-in telemetry around completed AG-UI runs', async () => { + const stub = new StubAgent(); + const seen: AgentRuntimeTelemetryPayload[] = []; + const a = toAgent(stub as unknown as AbstractAgent, { + telemetry: (payload) => seen.push(payload), + }); + + await a.submit({ message: 'hi' }); + + expect(seen.map((payload) => payload.event)).toEqual([ + 'ngaf:runtime_instance_created', + 'ngaf:runtime_request_created', + 'ngaf:stream_started', + 'ngaf:stream_ended', + ]); + expect(seen[0].properties).toEqual({ transport: 'ag-ui', surface: 'to_agent' }); + expect(seen[1].properties).toEqual({ transport: 'ag-ui', surface: 'to_agent', requestType: 'submit' }); + expect(seen[2].properties).toEqual({ transport: 'ag-ui', surface: 'to_agent' }); + expect(seen[3].properties).toEqual({ + transport: 'ag-ui', + surface: 'to_agent', + durationMs: expect.any(Number), + }); + }); + + it('emits opt-in telemetry for AG-UI failures without error messages', async () => { + const stub = new StubAgent(); + const seen: AgentRuntimeTelemetryPayload[] = []; + const a = toAgent(stub as unknown as AbstractAgent, { + telemetry: (payload) => seen.push(payload), + }); + stub.runAgent.mockRejectedValueOnce(new SyntaxError('private app state')); + + await a.submit({ message: 'hi' }); + + const errored = seen.find((payload) => payload.event === 'ngaf:stream_errored'); + expect(errored?.properties).toEqual({ + transport: 'ag-ui', + surface: 'to_agent', + durationMs: expect.any(Number), + errorClass: 'SyntaxError', + }); + expect(JSON.stringify(seen)).not.toContain('private app state'); + }); + it('stop() calls source.abortRun()', async () => { const stub = new StubAgent(); const a = toAgent(stub as unknown as AbstractAgent); diff --git a/libs/ag-ui/src/lib/to-agent.ts b/libs/ag-ui/src/lib/to-agent.ts index 2b5492482..148496c5f 100644 --- a/libs/ag-ui/src/lib/to-agent.ts +++ b/libs/ag-ui/src/lib/to-agent.ts @@ -4,10 +4,45 @@ import { Subject } from 'rxjs'; import type { AbstractAgent } from '@ag-ui/client'; import type { Agent, Message, AgentStatus, ToolCall, AgentEvent, + AgentRuntimeTelemetryEvent, + AgentRuntimeTelemetryProperties, + AgentRuntimeTelemetrySink, AgentSubmitInput, AgentSubmitOptions, } from '@ngaf/chat'; import { reduceEvent, type ReducerStore } from './reducer'; +export interface ToAgentOptions { + /** Optional app-owned telemetry sink. No telemetry is emitted unless this is provided. */ + telemetry?: AgentRuntimeTelemetrySink | false; +} + +function captureAgentRuntimeTelemetry( + sink: AgentRuntimeTelemetrySink | false | undefined, + event: AgentRuntimeTelemetryEvent, + properties: AgentRuntimeTelemetryProperties, +): void { + if (!sink) return; + try { + void Promise.resolve(sink({ event, properties })).catch(() => undefined); + } catch { + // Keep telemetry side effects isolated from adapter control flow. + } +} + +function agentRuntimeTelemetryErrorClass(error: unknown): string { + if (error instanceof Error) return error.name || error.constructor.name || 'Error'; + if ( + error + && typeof error === 'object' + && 'name' in error + && typeof error.name === 'string' + && error.name.length > 0 + ) { + return error.name; + } + return 'UnknownError'; +} + /** * Wraps an AG-UI AbstractAgent into the runtime-neutral Agent contract. * @@ -22,7 +57,7 @@ import { reduceEvent, type ReducerStore } from './reducer'; * agent instance they constructed. The subscriber registered via * source.subscribe() will fire for the lifetime of source. */ -export function toAgent(source: AbstractAgent): Agent { +export function toAgent(source: AbstractAgent, options: ToAgentOptions = {}): Agent { const store: ReducerStore = { messages: signal([]), status: signal('idle'), @@ -32,6 +67,45 @@ export function toAgent(source: AbstractAgent): Agent { state: signal>({}), events$: new Subject(), }; + const telemetryProperties = { transport: 'ag-ui' as const, surface: 'to_agent' }; + let activeRun: { startedAt: number; errored: boolean } | null = null; + + captureAgentRuntimeTelemetry( + options.telemetry, + 'ngaf:runtime_instance_created', + telemetryProperties, + ); + + function startRunTelemetry(requestType: string): { startedAt: number; errored: boolean } { + const run = { startedAt: Date.now(), errored: false }; + activeRun = run; + captureAgentRuntimeTelemetry(options.telemetry, 'ngaf:runtime_request_created', { + ...telemetryProperties, + requestType, + }); + captureAgentRuntimeTelemetry(options.telemetry, 'ngaf:stream_started', telemetryProperties); + return run; + } + + function finishRunTelemetry(run: { startedAt: number; errored: boolean }): void { + if (run.errored) return; + captureAgentRuntimeTelemetry(options.telemetry, 'ngaf:stream_ended', { + ...telemetryProperties, + durationMs: Date.now() - run.startedAt, + }); + if (activeRun === run) activeRun = null; + } + + function failRunTelemetry(error: unknown, run = activeRun): void { + if (!run || run.errored) return; + run.errored = true; + captureAgentRuntimeTelemetry(options.telemetry, 'ngaf:stream_errored', { + ...telemetryProperties, + durationMs: Date.now() - run.startedAt, + errorClass: agentRuntimeTelemetryErrorClass(error), + }); + if (activeRun === run) activeRun = null; + } // Tap all events from the source agent via the AgentSubscriber API. // This subscription lives for the lifetime of `source`. @@ -43,6 +117,7 @@ export function toAgent(source: AbstractAgent): Agent { store.status.set('error'); store.isLoading.set(false); store.error.set(error); + failRunTelemetry(error); }, }); @@ -65,14 +140,17 @@ export function toAgent(source: AbstractAgent): Agent { source.addMessage(userMsg as Parameters[0]); } + const run = startRunTelemetry('submit'); try { await source.runAgent(); + finishRunTelemetry(run); } catch (err) { // If the run was aborted via stop(), abortRun() resolves the promise // rather than rejecting — but catch any unexpected errors here. store.status.set('error'); store.isLoading.set(false); store.error.set(err); + failRunTelemetry(err, run); } }, @@ -113,12 +191,15 @@ export function toAgent(source: AbstractAgent): Agent { // message in `trimmed` becomes the active prompt for the next run. source.setMessages(trimmed as Parameters[0]); + const run = startRunTelemetry('regenerate'); try { await source.runAgent(); + finishRunTelemetry(run); } catch (err) { store.status.set('error'); store.isLoading.set(false); store.error.set(err); + failRunTelemetry(err, run); } }, }; diff --git a/libs/ag-ui/src/public-api.ts b/libs/ag-ui/src/public-api.ts index b51fedb08..217478bdf 100644 --- a/libs/ag-ui/src/public-api.ts +++ b/libs/ag-ui/src/public-api.ts @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT export { toAgent } from './lib/to-agent'; +export type { ToAgentOptions } from './lib/to-agent'; export { provideAgUiAgent, AG_UI_AGENT, injectAgUiAgent } from './lib/provide-ag-ui-agent'; export type { AgUiAgentConfig } from './lib/provide-ag-ui-agent'; export { FakeAgent } from './lib/testing/fake-agent'; diff --git a/libs/chat/src/lib/agent/index.ts b/libs/chat/src/lib/agent/index.ts index e9b08cb5f..81f528f5a 100644 --- a/libs/chat/src/lib/agent/index.ts +++ b/libs/chat/src/lib/agent/index.ts @@ -16,3 +16,9 @@ export type { } from './agent-event'; export type { AgentCheckpoint } from './agent-checkpoint'; export type { AgentWithHistory } from './agent-with-history'; +export type { + AgentRuntimeTelemetryEvent, + AgentRuntimeTelemetryPayload, + AgentRuntimeTelemetryProperties, + AgentRuntimeTelemetrySink, +} from './runtime-telemetry'; diff --git a/libs/chat/src/lib/agent/runtime-telemetry.ts b/libs/chat/src/lib/agent/runtime-telemetry.ts new file mode 100644 index 000000000..7532de0b3 --- /dev/null +++ b/libs/chat/src/lib/agent/runtime-telemetry.ts @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: MIT + +export type AgentRuntimeTelemetryEvent = + | 'ngaf:runtime_instance_created' + | 'ngaf:runtime_request_created' + | 'ngaf:stream_started' + | 'ngaf:stream_ended' + | 'ngaf:stream_errored'; + +export interface AgentRuntimeTelemetryProperties { + transport: 'langgraph' | 'ag-ui' | 'custom' | string; + surface?: string; + requestType?: string; + provider?: string; + model?: string; + durationMs?: number; + errorClass?: string; +} + +export interface AgentRuntimeTelemetryPayload { + event: AgentRuntimeTelemetryEvent; + properties: AgentRuntimeTelemetryProperties; +} + +export type AgentRuntimeTelemetrySink = (payload: AgentRuntimeTelemetryPayload) => void | Promise; diff --git a/libs/chat/src/public-api.ts b/libs/chat/src/public-api.ts index aa2c6572e..d33020a7d 100644 --- a/libs/chat/src/public-api.ts +++ b/libs/chat/src/public-api.ts @@ -24,6 +24,10 @@ export type { AgentStateUpdateEvent, AgentCustomEvent, AgentCheckpoint, + AgentRuntimeTelemetryEvent, + AgentRuntimeTelemetryPayload, + AgentRuntimeTelemetryProperties, + AgentRuntimeTelemetrySink, } from './lib/agent'; export { isUserMessage, diff --git a/libs/langgraph/src/lib/agent.types.ts b/libs/langgraph/src/lib/agent.types.ts index c9f658bad..378767cee 100644 --- a/libs/langgraph/src/lib/agent.types.ts +++ b/libs/langgraph/src/lib/agent.types.ts @@ -20,7 +20,12 @@ import type { SubmitOptions, } from '@langchain/langgraph-sdk/ui'; import type { BaseMessage, AIMessage as CoreAIMessage } from '@langchain/core/messages'; -import type { AgentSubmitInput, AgentSubmitOptions, AgentWithHistory } from '@ngaf/chat'; +import type { + AgentRuntimeTelemetrySink, + AgentSubmitInput, + AgentSubmitOptions, + AgentWithHistory, +} from '@ngaf/chat'; import type { AgentLifecycle } from './lifecycle'; // Re-export SDK types so consumers don't need to import from langgraph-sdk directly @@ -251,6 +256,8 @@ export interface AgentOptions { toMessage?: (msg: unknown) => BaseMessage; /** Custom transport. Defaults to FetchStreamTransport. */ transport?: AgentTransport; + /** Optional app-owned telemetry sink. No telemetry is emitted unless this is provided. */ + telemetry?: AgentRuntimeTelemetrySink | false; /** When true, subagent messages are filtered from the main messages signal. */ filterSubagentMessages?: boolean; /** Tool names that indicate a subagent invocation. */ diff --git a/libs/langgraph/src/lib/internals/stream-manager.bridge.spec.ts b/libs/langgraph/src/lib/internals/stream-manager.bridge.spec.ts index b5333ab15..a1fa58456 100644 --- a/libs/langgraph/src/lib/internals/stream-manager.bridge.spec.ts +++ b/libs/langgraph/src/lib/internals/stream-manager.bridge.spec.ts @@ -3,6 +3,7 @@ import { BehaviorSubject, Subject } from 'rxjs'; import { createStreamManagerBridge } from './stream-manager.bridge'; import { MockAgentTransport } from '../transport/mock-stream.transport'; import { ResourceStatus, AgentTransport, StreamSubjects, CustomStreamEvent, StreamEvent } from '../agent.types'; +import type { AgentRuntimeTelemetryPayload } from '@ngaf/chat'; import type { ThreadState } from '@langchain/langgraph-sdk'; import { of } from 'rxjs'; import { readFileSync } from 'node:fs'; @@ -149,6 +150,81 @@ describe('createStreamManagerBridge', () => { destroy$.next(); }); + it('emits opt-in telemetry around completed LangGraph streams', async () => { + const seen: AgentRuntimeTelemetryPayload[] = []; + const transport: AgentTransport = { + async *stream() { + yield { type: 'values', values: { ok: true } }; + }, + }; + const subjects = makeSubjects(); + const destroy$ = new Subject(); + const bridge = createStreamManagerBridge({ + options: { + apiUrl: '', + assistantId: 'test', + transport, + telemetry: (payload) => seen.push(payload), + }, + subjects, + threadId$: of('thread-1'), + destroy$: destroy$.asObservable(), + }); + + await bridge.submit({ messages: [] }); + + expect(seen.map((payload) => payload.event)).toEqual([ + 'ngaf:runtime_instance_created', + 'ngaf:runtime_request_created', + 'ngaf:stream_started', + 'ngaf:stream_ended', + ]); + expect(seen[0].properties).toEqual({ transport: 'langgraph', surface: 'agent' }); + expect(seen[1].properties).toEqual({ transport: 'langgraph', surface: 'agent', requestType: 'submit' }); + expect(seen[2].properties).toEqual({ transport: 'langgraph', surface: 'agent' }); + expect(seen[3].properties).toEqual({ + transport: 'langgraph', + surface: 'agent', + durationMs: expect.any(Number), + }); + destroy$.next(); + }); + + it('emits opt-in telemetry for LangGraph stream failures without error messages', async () => { + const seen: AgentRuntimeTelemetryPayload[] = []; + const transport: AgentTransport = { + async *stream() { + yield* []; + throw new TypeError('secret prompt fragment'); + }, + }; + const subjects = makeSubjects(); + const destroy$ = new Subject(); + const bridge = createStreamManagerBridge({ + options: { + apiUrl: '', + assistantId: 'test', + transport, + telemetry: (payload) => seen.push(payload), + }, + subjects, + threadId$: of('thread-1'), + destroy$: destroy$.asObservable(), + }); + + await bridge.submit({ messages: [] }); + + const errored = seen.find((payload) => payload.event === 'ngaf:stream_errored'); + expect(errored?.properties).toEqual({ + transport: 'langgraph', + surface: 'agent', + durationMs: expect.any(Number), + errorClass: 'TypeError', + }); + expect(JSON.stringify(seen)).not.toContain('secret prompt fragment'); + destroy$.next(); + }); + it('loads history when initialized with a thread id', async () => { const history = [makeThreadState('checkpoint-1')]; const historyCalls: string[] = []; diff --git a/libs/langgraph/src/lib/internals/stream-manager.bridge.ts b/libs/langgraph/src/lib/internals/stream-manager.bridge.ts index ea5dbad3c..d6115bcf8 100644 --- a/libs/langgraph/src/lib/internals/stream-manager.bridge.ts +++ b/libs/langgraph/src/lib/internals/stream-manager.bridge.ts @@ -15,6 +15,11 @@ import { import { FetchStreamTransport } from '../transport/fetch-stream.transport'; import { BagTemplate } from '@langchain/langgraph-sdk'; import { getToolCallsWithResults } from '@langchain/langgraph-sdk/utils'; +import type { + AgentRuntimeTelemetryEvent, + AgentRuntimeTelemetryProperties, + AgentRuntimeTelemetrySink, +} from '@ngaf/chat'; import { SubagentTracker, TrackedSubagent, @@ -40,6 +45,33 @@ function lgTrace(...args: unknown[]): void { } } +function captureAgentRuntimeTelemetry( + sink: AgentRuntimeTelemetrySink | false | undefined, + event: AgentRuntimeTelemetryEvent, + properties: AgentRuntimeTelemetryProperties, +): void { + if (!sink) return; + try { + void Promise.resolve(sink({ event, properties })).catch(() => undefined); + } catch { + // Keep telemetry side effects isolated from stream control flow. + } +} + +function agentRuntimeTelemetryErrorClass(error: unknown): string { + if (error instanceof Error) return error.name || error.constructor.name || 'Error'; + if ( + error + && typeof error === 'object' + && 'name' in error + && typeof error.name === 'string' + && error.name.length > 0 + ) { + return error.name; + } + return 'UnknownError'; +} + export interface StreamManagerBridgeOptions { options: AgentOptions; subjects: StreamSubjects; @@ -87,6 +119,19 @@ export function createStreamManagerBridge { abortController = new AbortController(); + const startedAt = Date.now(); + captureRuntimeRequestTelemetry('join_queued'); + captureAgentRuntimeTelemetry(options.telemetry, 'ngaf:stream_started', telemetryProperties); subjects.custom$.next([]); subjects.toolProgress$.next([]); toolProgressMap.clear(); @@ -305,16 +354,32 @@ export function createStreamManagerBridge { + async function runStream( + payload: unknown, + opts?: LangGraphSubmitOptions, + requestType = 'submit', + ): Promise { abortController?.abort(); abortController = new AbortController(); + const startedAt = Date.now(); + captureRuntimeRequestTelemetry(requestType); + captureAgentRuntimeTelemetry(options.telemetry, 'ngaf:stream_started', telemetryProperties); subjects.status$.next(ResourceStatus.Loading); subjects.error$.next(undefined); @@ -362,6 +427,10 @@ export function createStreamManagerBridge { if (lastPayload !== null) { - await runStream(lastPayload, lastOptions); + await runStream(lastPayload, lastOptions, 'resubmit'); } }, diff --git a/libs/telemetry/README.md b/libs/telemetry/README.md index 6af942247..4683a6b21 100644 --- a/libs/telemetry/README.md +++ b/libs/telemetry/README.md @@ -12,6 +12,7 @@ import { provideNgafTelemetry } from '@ngaf/telemetry/browser'; // Node (server adapters) import { captureRuntimeInstanceCreated, + captureRuntimeRequestCreated, captureStreamStarted, captureStreamEnded, captureStreamErrored, @@ -31,11 +32,12 @@ The single telemetry surface for `@ngaf/*`. It exists so we can answer "how is C **Telemetered by default (Node, opt-out):** - `ngaf:postinstall` — fires once per dependency/global install of a published `@ngaf/*` package. Properties: package name, package version, Node version, OS, CPU architecture, package manager name/version, installer-reported Node/OS/architecture, workspace/global install flags when npm exposes them, sample weight. It uses a per-process anonymous id. No project path, no raw environment variables, no dependency tree, no installer IP address. - `ngaf:runtime_instance_created` — server adapters (LangGraph, AG-UI) call this when they spin up. Properties: which transport, which model provider (string), Angular peer version. **No API keys**, no endpoint hostnames, no user data. +- `ngaf:runtime_request_created` — server adapters call this when they create a transport request. Properties: transport, request type, provider, model. No prompts, thread IDs, assistant IDs, endpoint URLs, or headers. - `ngaf:stream_started` / `ngaf:stream_ended` / `ngaf:stream_errored` — per-request lifecycle on server adapters. Properties: provider, model name, duration, error class. No prompts, no completions, no message content. **Telemetered only on explicit opt-in (Browser):** - Nothing fires unless the consumer calls `provideNgafTelemetry({ enabled: true, sink })` or `provideNgafTelemetry({ enabled: true, endpoint })` in their root providers. -- When opted in: `ngaf:browser_provided`, `ngaf:browser_chat_init`, and browser-side runtime lifecycle events explicitly captured by the app (`ngaf:runtime_instance_created`, `ngaf:stream_started`, `ngaf:stream_ended`, `ngaf:stream_errored`). Anonymous, no message content. +- When opted in: `ngaf:browser_provided`, `ngaf:browser_chat_init`, and browser-side runtime lifecycle events explicitly captured by the app (`ngaf:runtime_instance_created`, `ngaf:runtime_request_created`, `ngaf:stream_started`, `ngaf:stream_ended`, `ngaf:stream_errored`). Anonymous, no message content. **Never telemetered (by anyone, at any time):** - Message content (user prompts, model completions, tool call inputs/outputs). diff --git a/libs/telemetry/scripts/verify-install-telemetry.mjs b/libs/telemetry/scripts/verify-install-telemetry.mjs new file mode 100644 index 000000000..7e78fa1aa --- /dev/null +++ b/libs/telemetry/scripts/verify-install-telemetry.mjs @@ -0,0 +1,49 @@ +#!/usr/bin/env node +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const TELEMETRY_DEP = '@ngaf/telemetry'; +const POSTINSTALL = 'ngaf-telemetry-postinstall || true'; + +export function verifyInstallTelemetryManifest(pkg, manifestPath = 'package.json') { + if (pkg?.name === TELEMETRY_DEP) return; + if (typeof pkg?.name !== 'string' || !pkg.name.startsWith('@ngaf/')) { + throw new Error(`${manifestPath} is not an @ngaf package manifest`); + } + + const actualDep = pkg.dependencies?.[TELEMETRY_DEP]; + if (actualDep !== '*') { + throw new Error(`${pkg.name} is missing dependencies["${TELEMETRY_DEP}"] = "*" in ${manifestPath}`); + } + + const actualPostinstall = pkg.scripts?.postinstall; + if (actualPostinstall !== POSTINSTALL) { + throw new Error(`${pkg.name} is missing scripts.postinstall = "${POSTINSTALL}" in ${manifestPath}`); + } +} + +export async function verifyInstallTelemetryManifests(packageRoots) { + if (packageRoots.length === 0) { + throw new Error('Usage: node libs/telemetry/scripts/verify-install-telemetry.mjs [...]'); + } + + for (const root of packageRoots) { + const manifestPath = join(root, 'package.json'); + const pkg = JSON.parse(await readFile(manifestPath, 'utf8')); + verifyInstallTelemetryManifest(pkg, manifestPath); + } +} + +async function main() { + try { + await verifyInstallTelemetryManifests(process.argv.slice(2)); + } catch (err) { + console.error(err instanceof Error ? err.message : err); + process.exit(1); + } +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + await main(); +} diff --git a/libs/telemetry/scripts/verify-install-telemetry.spec.mjs b/libs/telemetry/scripts/verify-install-telemetry.spec.mjs new file mode 100644 index 000000000..f1c533232 --- /dev/null +++ b/libs/telemetry/scripts/verify-install-telemetry.spec.mjs @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest'; + +import { verifyInstallTelemetryManifest } from './verify-install-telemetry.mjs'; + +describe('verify-install-telemetry', () => { + it('requires publishable packages to depend on telemetry and run the postinstall hook', () => { + expect(() => verifyInstallTelemetryManifest({ + name: '@ngaf/chat', + dependencies: {}, + scripts: {}, + }, '/tmp/dist/libs/chat/package.json')).toThrow(/@ngaf\/chat/); + + expect(() => verifyInstallTelemetryManifest({ + name: '@ngaf/chat', + dependencies: { '@ngaf/telemetry': '*' }, + scripts: { postinstall: 'ngaf-telemetry-postinstall || true' }, + }, '/tmp/dist/libs/chat/package.json')).not.toThrow(); + }); + + it('does not require the telemetry package to install itself', () => { + expect(() => verifyInstallTelemetryManifest({ + name: '@ngaf/telemetry', + dependencies: {}, + scripts: {}, + }, '/tmp/dist/libs/telemetry/package.json')).not.toThrow(); + }); +}); diff --git a/libs/telemetry/src/browser/service.spec.ts b/libs/telemetry/src/browser/service.spec.ts index d9804d054..a0e9bc72f 100644 --- a/libs/telemetry/src/browser/service.spec.ts +++ b/libs/telemetry/src/browser/service.spec.ts @@ -69,6 +69,34 @@ describe('NgafTelemetryService', () => { }); }); + test('captureRuntimeRequestCreated() records request type through the neutral sink', async () => { + const sink = vi.fn(); + TestBed.configureTestingModule({ + providers: [ + { provide: NGAF_TELEMETRY_CONFIG, useValue: { enabled: true, sink } }, + NgafTelemetryService, + ], + }); + const svc = TestBed.inject(NgafTelemetryService); + + await svc.captureRuntimeRequestCreated({ + transport: 'langgraph', + surface: 'canonical_demo', + requestType: 'submit', + }); + + expect(sink).toHaveBeenCalledWith({ + event: 'ngaf:runtime_request_created', + properties: { + transport: 'langgraph', + surface: 'canonical_demo', + requestType: 'submit', + sample_weight: 1, + }, + }); + }); + + test('capture() posts neutral event payloads to a configured endpoint', async () => { const fetchMock = vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('{}', { status: 202 })); TestBed.configureTestingModule({ diff --git a/libs/telemetry/src/browser/service.ts b/libs/telemetry/src/browser/service.ts index f524b8924..284a22ee2 100644 --- a/libs/telemetry/src/browser/service.ts +++ b/libs/telemetry/src/browser/service.ts @@ -80,6 +80,10 @@ export class NgafTelemetryService { return this.capture('ngaf:runtime_instance_created', { ...input }); } + captureRuntimeRequestCreated(input: NgafBrowserRuntimeTelemetry & { requestType: string }): Promise { + return this.capture('ngaf:runtime_request_created', { ...input }); + } + captureStreamStarted(input: NgafBrowserStreamTelemetry): Promise { return this.capture('ngaf:stream_started', { ...input }); } diff --git a/libs/telemetry/src/browser/tokens.ts b/libs/telemetry/src/browser/tokens.ts index 4982f8b33..9176222bd 100644 --- a/libs/telemetry/src/browser/tokens.ts +++ b/libs/telemetry/src/browser/tokens.ts @@ -4,6 +4,7 @@ export type NgafTelemetryEvent = | 'ngaf:browser_provided' | 'ngaf:browser_chat_init' | 'ngaf:runtime_instance_created' + | 'ngaf:runtime_request_created' | 'ngaf:stream_started' | 'ngaf:stream_ended' | 'ngaf:stream_errored'; diff --git a/libs/telemetry/src/node/adapter.spec.ts b/libs/telemetry/src/node/adapter.spec.ts index cb300a36a..063b895c8 100644 --- a/libs/telemetry/src/node/adapter.spec.ts +++ b/libs/telemetry/src/node/adapter.spec.ts @@ -7,6 +7,7 @@ vi.mock('./client', () => ({ import { captureEvent } from './client'; import { captureRuntimeInstanceCreated, + captureRuntimeRequestCreated, captureStreamStarted, captureStreamEnded, captureStreamErrored, @@ -36,6 +37,24 @@ describe('adapter helpers', () => { ); }); + test('captureRuntimeRequestCreated records request type without content identifiers', async () => { + await captureRuntimeRequestCreated({ + transport: 'langgraph', + requestType: 'run', + provider: 'openai', + model: 'gpt-4', + }); + expect(captureEvent).toHaveBeenCalledWith( + 'ngaf:runtime_request_created', + expect.objectContaining({ + transport: 'langgraph', + requestType: 'run', + provider: 'openai', + model: 'gpt-4', + }), + ); + }); + test('captureStreamEnded records duration', async () => { await captureStreamEnded({ provider: 'openai', model: 'gpt-4', durationMs: 1234 }); expect(captureEvent).toHaveBeenCalledWith( diff --git a/libs/telemetry/src/node/adapter.ts b/libs/telemetry/src/node/adapter.ts index 631a65d94..df67ea9c8 100644 --- a/libs/telemetry/src/node/adapter.ts +++ b/libs/telemetry/src/node/adapter.ts @@ -14,6 +14,13 @@ export interface StreamTelemetry { durationMs?: number; } +export interface RuntimeRequestTelemetry { + transport: string; + requestType: string; + provider?: string; + model?: string; +} + async function safe(fn: () => Promise): Promise { try { await fn(); } catch { /* silent fail */ } } @@ -26,6 +33,10 @@ export async function captureRuntimeInstanceCreated(input: RuntimeInstanceTeleme }); } +export async function captureRuntimeRequestCreated(input: RuntimeRequestTelemetry): Promise { + await safe(() => captureEvent('ngaf:runtime_request_created', { ...input })); +} + export async function captureStreamStarted(input: StreamTelemetry): Promise { await safe(() => captureEvent('ngaf:stream_started', { ...input })); } diff --git a/libs/telemetry/src/node/index.ts b/libs/telemetry/src/node/index.ts index 90427d9be..1bfbcdc75 100644 --- a/libs/telemetry/src/node/index.ts +++ b/libs/telemetry/src/node/index.ts @@ -3,8 +3,9 @@ export { capturePostinstall, captureEvent } from './client.js'; export type { CaptureResult } from './client.js'; export { captureRuntimeInstanceCreated, + captureRuntimeRequestCreated, captureStreamStarted, captureStreamEnded, captureStreamErrored, } from './adapter.js'; -export type { RuntimeInstanceTelemetry, StreamTelemetry } from './adapter.js'; +export type { RuntimeInstanceTelemetry, RuntimeRequestTelemetry, StreamTelemetry } from './adapter.js'; diff --git a/libs/telemetry/src/shared/events.ts b/libs/telemetry/src/shared/events.ts index 7aabec319..6c4fa3500 100644 --- a/libs/telemetry/src/shared/events.ts +++ b/libs/telemetry/src/shared/events.ts @@ -1,6 +1,7 @@ export type NgafNodeEvent = | 'ngaf:postinstall' | 'ngaf:runtime_instance_created' + | 'ngaf:runtime_request_created' | 'ngaf:stream_started' | 'ngaf:stream_ended' | 'ngaf:stream_errored'; diff --git a/libs/telemetry/vite.config.mts b/libs/telemetry/vite.config.mts index 7aabd4c33..0fe8aae4d 100644 --- a/libs/telemetry/vite.config.mts +++ b/libs/telemetry/vite.config.mts @@ -6,7 +6,7 @@ export default defineConfig({ test: { environment: 'node', globals: true, - include: ['src/**/*.spec.ts'], + include: ['src/**/*.spec.ts', 'scripts/**/*.spec.mjs'], setupFiles: ['src/test-setup.ts'], }, }); diff --git a/nx.json b/nx.json index c9844a011..1d018b37a 100644 --- a/nx.json +++ b/nx.json @@ -58,7 +58,7 @@ } }, "version": { - "preVersionCommand": "npx nx run-many -t build --projects=chat,langgraph,ag-ui,render,a2ui,licensing,telemetry && node libs/telemetry/scripts/apply-install-telemetry.mjs dist/libs/chat dist/libs/langgraph dist/libs/ag-ui dist/libs/render dist/libs/a2ui dist/libs/licensing", + "preVersionCommand": "npx nx run-many -t build --projects=chat,langgraph,ag-ui,render,a2ui,licensing,telemetry && node libs/telemetry/scripts/apply-install-telemetry.mjs dist/libs/chat dist/libs/langgraph dist/libs/ag-ui dist/libs/render dist/libs/a2ui dist/libs/licensing && node libs/telemetry/scripts/verify-install-telemetry.mjs dist/libs/chat dist/libs/langgraph dist/libs/ag-ui dist/libs/render dist/libs/a2ui dist/libs/licensing dist/libs/telemetry", "updateDependents": "auto", "preserveLocalDependencyProtocols": true, "currentVersionResolver": "git-tag", diff --git a/scripts/langgraph-proxy.spec.ts b/scripts/langgraph-proxy.spec.ts index 8b56bd230..6b7bef344 100644 --- a/scripts/langgraph-proxy.spec.ts +++ b/scripts/langgraph-proxy.spec.ts @@ -84,6 +84,47 @@ describe('createProxyHandler', () => { expect(res.send).toHaveBeenCalledWith('{"ok":true}'); }); + it('forwards canonical demo runtime request telemetry without content fields', async () => { + const fetchMock = vi.spyOn(global, 'fetch').mockResolvedValue( + new Response('{"ok":true}', { status: 202, headers: { 'content-type': 'application/json' } }), + ); + const handler = createProxyHandler({ + backendUrl: DEFAULT_BACKEND, + telemetryIngestUrl: 'https://cacheplane.ai/api/ingest', + }); + const res = makeRes(); + + await handler({ + method: 'POST', + headers: { host: 'demo.cacheplane.ai', 'content-type': 'application/json' }, + body: { + event: 'ngaf:runtime_request_created', + distinctId: 'browser:test', + properties: { + transport: 'langgraph', + surface: 'canonical_demo', + requestType: 'submit', + }, + }, + url: '/api/ingest', + query: {}, + } as never, res as never); + + const forwarded = JSON.parse(String((fetchMock.mock.calls[0]![1] as RequestInit).body)); + expect(forwarded).toEqual({ + event: 'ngaf:runtime_request_created', + distinctId: 'browser:test', + properties: { + transport: 'langgraph', + surface: 'canonical_demo', + requestType: 'submit', + }, + key: 'phc_public_cacheplane_telemetry', + }); + expect(JSON.stringify(forwarded)).not.toMatch(/messages|threadId|assistantId|apiUrl/); + }); + + it('responds 204 to OPTIONS preflight with CORS headers', async () => { const handler = createProxyHandler({ backendUrl: DEFAULT_BACKEND }); const res = makeRes(); diff --git a/tools/posthog/dashboards/runtime-telemetry.json b/tools/posthog/dashboards/runtime-telemetry.json new file mode 100644 index 000000000..b61e33d18 --- /dev/null +++ b/tools/posthog/dashboards/runtime-telemetry.json @@ -0,0 +1,28 @@ +{ + "slug": "runtime-telemetry", + "posthog_id": null, + "name": "GTM · Runtime telemetry", + "description": "Opt-in browser and opt-out Node runtime telemetry for @ngaf/* adapters.", + "tags": [ + "gtm", + "runtime-telemetry", + "phase-1" + ], + "tiles": [ + { + "insight": "runtime-instances-by-transport" + }, + { + "insight": "runtime-requests-by-transport" + }, + { + "insight": "runtime-stream-starts-by-transport" + }, + { + "insight": "runtime-stream-ends-by-transport" + }, + { + "insight": "runtime-stream-errors-by-transport" + } + ] +} diff --git a/tools/posthog/insights/runtime-instances-by-transport.json b/tools/posthog/insights/runtime-instances-by-transport.json new file mode 100644 index 000000000..e4355999e --- /dev/null +++ b/tools/posthog/insights/runtime-instances-by-transport.json @@ -0,0 +1,16 @@ +{ + "slug": "runtime-instances-by-transport", + "posthog_id": null, + "kind": "trends", + "name": "Runtime instances by transport", + "events": [ + { + "event": "ngaf:runtime_instance_created", + "math": "total" + } + ], + "breakdown": "transport", + "breakdown_limit": 10, + "interval": "day", + "date_from": "-30d" +} diff --git a/tools/posthog/insights/runtime-requests-by-transport.json b/tools/posthog/insights/runtime-requests-by-transport.json new file mode 100644 index 000000000..08cd28344 --- /dev/null +++ b/tools/posthog/insights/runtime-requests-by-transport.json @@ -0,0 +1,16 @@ +{ + "slug": "runtime-requests-by-transport", + "posthog_id": null, + "kind": "trends", + "name": "Runtime requests by transport", + "events": [ + { + "event": "ngaf:runtime_request_created", + "math": "total" + } + ], + "breakdown": "transport", + "breakdown_limit": 10, + "interval": "day", + "date_from": "-30d" +} diff --git a/tools/posthog/insights/runtime-stream-ends-by-transport.json b/tools/posthog/insights/runtime-stream-ends-by-transport.json new file mode 100644 index 000000000..a14b21f67 --- /dev/null +++ b/tools/posthog/insights/runtime-stream-ends-by-transport.json @@ -0,0 +1,16 @@ +{ + "slug": "runtime-stream-ends-by-transport", + "posthog_id": null, + "kind": "trends", + "name": "Runtime stream completions by transport", + "events": [ + { + "event": "ngaf:stream_ended", + "math": "total" + } + ], + "breakdown": "transport", + "breakdown_limit": 10, + "interval": "day", + "date_from": "-30d" +} diff --git a/tools/posthog/insights/runtime-stream-errors-by-transport.json b/tools/posthog/insights/runtime-stream-errors-by-transport.json new file mode 100644 index 000000000..d67a3869b --- /dev/null +++ b/tools/posthog/insights/runtime-stream-errors-by-transport.json @@ -0,0 +1,16 @@ +{ + "slug": "runtime-stream-errors-by-transport", + "posthog_id": null, + "kind": "trends", + "name": "Runtime stream errors by transport", + "events": [ + { + "event": "ngaf:stream_errored", + "math": "total" + } + ], + "breakdown": "transport", + "breakdown_limit": 10, + "interval": "day", + "date_from": "-30d" +} diff --git a/tools/posthog/insights/runtime-stream-starts-by-transport.json b/tools/posthog/insights/runtime-stream-starts-by-transport.json new file mode 100644 index 000000000..9162a3b23 --- /dev/null +++ b/tools/posthog/insights/runtime-stream-starts-by-transport.json @@ -0,0 +1,16 @@ +{ + "slug": "runtime-stream-starts-by-transport", + "posthog_id": null, + "kind": "trends", + "name": "Runtime stream starts by transport", + "events": [ + { + "event": "ngaf:stream_started", + "math": "total" + } + ], + "breakdown": "transport", + "breakdown_limit": 10, + "interval": "day", + "date_from": "-30d" +}