From a67d527feba52e1bc48dbbb8ca1222e5a12804ee Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Mon, 30 Mar 2026 11:17:31 +0200 Subject: [PATCH 01/47] Add track() API for custom event tracking --- docs/track.md | 48 +++++++++++++++++++++++++++++++++ library/agent/Agent.ts | 21 +++++++++++++++ library/agent/api/Event.ts | 20 +++++++++++++- library/agent/context/track.ts | 49 ++++++++++++++++++++++++++++++++++ library/index.ts | 3 +++ 5 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 docs/track.md create mode 100644 library/agent/context/track.ts diff --git a/docs/track.md b/docs/track.md new file mode 100644 index 000000000..6c65d0335 --- /dev/null +++ b/docs/track.md @@ -0,0 +1,48 @@ +# Tracking events + +`track` lets you record things happening in your app — like failed logins, signups, or password resets. Zen sends these to Aikido so patterns can be detected, like someone failing to log in 50 times in a minute. + +```js +const Zen = require("@aikidosec/firewall"); + +app.post("/login", async (req, res) => { + const user = await authenticate(req.body.username, req.body.password); + + if (!user) { + Zen.track("user.login_failed"); + return res.status(401).json({ error: "Invalid credentials" }); + } + + Zen.setUser({ id: user.id }); + Zen.track("user.login_succeeded"); + res.json({ token: createToken(user) }); +}); +``` + +Zen automatically picks up the IP address, user agent, and current user (if you called [`setUser`](./user.md)) from the request — you don't need to pass those yourself. + +## More examples + +```js +Zen.track("user.signed_up"); +Zen.track("user.password_reset_requested"); +Zen.track("plan.invite_sent"); +Zen.track("payment.failed"); +``` + +## Naming events + +Use lowercase with dots to group related events: + +- `user.login_failed` +- `user.login_succeeded` +- `user.signed_up` +- `user.password_reset_requested` +- `payment.failed` +- `plan.invite_sent` + +## Things to know + +`track` only works inside an HTTP request. If you call it in a background job or a script, nothing gets sent and you'll see a warning in the console. + +If you haven't called `setUser` yet, the event still goes through — it just won't have a user ID attached. diff --git a/library/agent/Agent.ts b/library/agent/Agent.ts index fcf429167..3b32ed108 100644 --- a/library/agent/Agent.ts +++ b/library/agent/Agent.ts @@ -12,6 +12,7 @@ import type { AgentInfo, DetectedAttack, DetectedAttackWave, + TrackedEvent, } from "./api/Event"; import { Token } from "./api/Token"; import { Kind } from "./Attack"; @@ -712,4 +713,24 @@ export class Agent { this.pendingEvents.onAPICall(promise); } } + + onTrackEvent(event: TrackedEvent["event"]) { + if (!this.token) { + return; + } + + const tracked: TrackedEvent = { + type: "tracked_event", + event, + agent: this.getAgentInfo(), + time: Date.now(), + }; + + const promise = this.api + .report(this.token, tracked, this.timeoutInMS) + .catch(() => { + this.logger.log("Failed to report tracked event"); + }); + this.pendingEvents.onAPICall(promise); + } } diff --git a/library/agent/api/Event.ts b/library/agent/api/Event.ts index bc4a8181c..6436320aa 100644 --- a/library/agent/api/Event.ts +++ b/library/agent/api/Event.ts @@ -170,4 +170,22 @@ export type DetectedAttackWave = { time: number; }; -export type Event = Started | DetectedAttack | Heartbeat | DetectedAttackWave; +export type TrackedEvent = { + type: "tracked_event"; + event: { + name: string; + userId: string | undefined; + metadata: Record; + ipAddress: string | undefined; + userAgent: string | undefined; + }; + agent: AgentInfo; + time: number; +}; + +export type Event = + | Started + | DetectedAttack + | Heartbeat + | DetectedAttackWave + | TrackedEvent; diff --git a/library/agent/context/track.ts b/library/agent/context/track.ts new file mode 100644 index 000000000..91868de04 --- /dev/null +++ b/library/agent/context/track.ts @@ -0,0 +1,49 @@ +import { getInstance } from "../AgentSingleton"; +import { ContextStorage } from "./ContextStorage"; + +export function track(eventName: string): void { + const agent = getInstance(); + + if (!agent) { + return; + } + + if (typeof eventName !== "string" || eventName.length === 0) { + agent.log(`track(...) expects a non-empty string as event name.`); + return; + } + + const context = ContextStorage.getStore(); + if (!context) { + logWarningTrackCalledWithoutContext(); + return; + } + + const userAgent = + typeof context.headers["user-agent"] === "string" + ? context.headers["user-agent"] + : undefined; + + agent.onTrackEvent({ + name: eventName, + userId: context.user?.id, + metadata: {}, + ipAddress: context.remoteAddress, + userAgent, + }); +} + +let loggedWarningTrackCalledWithoutContext = false; + +function logWarningTrackCalledWithoutContext() { + if (loggedWarningTrackCalledWithoutContext) { + return; + } + + // oxlint-disable-next-line no-console + console.warn( + "track(...) was called without a context. The event will not be tracked. Make sure to call track(...) within an HTTP request." + ); + + loggedWarningTrackCalledWithoutContext = true; +} diff --git a/library/index.ts b/library/index.ts index b1a75baba..36b002be2 100644 --- a/library/index.ts +++ b/library/index.ts @@ -1,6 +1,7 @@ import isFirewallSupported from "./helpers/isFirewallSupported"; import shouldEnableFirewall from "./helpers/shouldEnableFirewall"; import { setUser } from "./agent/context/user"; +import { track } from "./agent/context/track"; import { markUnsafe } from "./agent/context/markUnsafe"; import { shouldBlockRequest } from "./middleware/shouldBlockRequest"; import { addExpressMiddleware } from "./middleware/express"; @@ -58,6 +59,7 @@ if (!isNewHookSystemUsed()) { export { setUser, + track, markUnsafe, shouldBlockRequest, addExpressMiddleware, @@ -77,6 +79,7 @@ export { // e.g. import Zen from '@aikidosec/firewall'; would not work without this, as Zen.setUser would be undefined export default { setUser, + track, markUnsafe, shouldBlockRequest, addExpressMiddleware, From 4c173f9eed98a7e43c431ccbc4b9c5dec12a5894 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Mon, 30 Mar 2026 11:20:26 +0200 Subject: [PATCH 02/47] Remove unused metadata field from TrackedEvent --- library/agent/api/Event.ts | 1 - library/agent/context/track.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/library/agent/api/Event.ts b/library/agent/api/Event.ts index 6436320aa..d92d1f315 100644 --- a/library/agent/api/Event.ts +++ b/library/agent/api/Event.ts @@ -175,7 +175,6 @@ export type TrackedEvent = { event: { name: string; userId: string | undefined; - metadata: Record; ipAddress: string | undefined; userAgent: string | undefined; }; diff --git a/library/agent/context/track.ts b/library/agent/context/track.ts index 91868de04..312c6e850 100644 --- a/library/agent/context/track.ts +++ b/library/agent/context/track.ts @@ -27,7 +27,6 @@ export function track(eventName: string): void { agent.onTrackEvent({ name: eventName, userId: context.user?.id, - metadata: {}, ipAddress: context.remoteAddress, userAgent, }); From eb3912e64643b2938d1f50071a10df0109ecd0c1 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Mon, 30 Mar 2026 13:51:04 +0200 Subject: [PATCH 03/47] Send tracked events to the realtime endpoint --- library/agent/Agent.ts | 19 +++++-------------- library/agent/api/Event.ts | 19 +------------------ library/agent/api/UserEventsAPI.ts | 26 ++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 32 deletions(-) create mode 100644 library/agent/api/UserEventsAPI.ts diff --git a/library/agent/Agent.ts b/library/agent/Agent.ts index 3b32ed108..f9ae97706 100644 --- a/library/agent/Agent.ts +++ b/library/agent/Agent.ts @@ -12,8 +12,8 @@ import type { AgentInfo, DetectedAttack, DetectedAttackWave, - TrackedEvent, } from "./api/Event"; +import { sendUserEvent, type UserEvent } from "./api/UserEventsAPI"; import { Token } from "./api/Token"; import { Kind } from "./Attack"; import { Endpoint } from "./Config"; @@ -714,23 +714,14 @@ export class Agent { } } - onTrackEvent(event: TrackedEvent["event"]) { + onTrackEvent(event: UserEvent) { if (!this.token) { return; } - const tracked: TrackedEvent = { - type: "tracked_event", - event, - agent: this.getAgentInfo(), - time: Date.now(), - }; - - const promise = this.api - .report(this.token, tracked, this.timeoutInMS) - .catch(() => { - this.logger.log("Failed to report tracked event"); - }); + const promise = sendUserEvent(this.token, event).catch(() => { + this.logger.log("Failed to report tracked event"); + }); this.pendingEvents.onAPICall(promise); } } diff --git a/library/agent/api/Event.ts b/library/agent/api/Event.ts index d92d1f315..bc4a8181c 100644 --- a/library/agent/api/Event.ts +++ b/library/agent/api/Event.ts @@ -170,21 +170,4 @@ export type DetectedAttackWave = { time: number; }; -export type TrackedEvent = { - type: "tracked_event"; - event: { - name: string; - userId: string | undefined; - ipAddress: string | undefined; - userAgent: string | undefined; - }; - agent: AgentInfo; - time: number; -}; - -export type Event = - | Started - | DetectedAttack - | Heartbeat - | DetectedAttackWave - | TrackedEvent; +export type Event = Started | DetectedAttack | Heartbeat | DetectedAttackWave; diff --git a/library/agent/api/UserEventsAPI.ts b/library/agent/api/UserEventsAPI.ts new file mode 100644 index 000000000..7dfa0c323 --- /dev/null +++ b/library/agent/api/UserEventsAPI.ts @@ -0,0 +1,26 @@ +import { fetch } from "../../helpers/fetch"; +import { getRealtimeURL } from "../realtime/getRealtimeURL"; +import type { Token } from "./Token"; + +export type UserEvent = { + name: string; + userId: string | undefined; + ipAddress: string | undefined; + userAgent: string | undefined; +}; + +export async function sendUserEvent( + token: Token, + event: UserEvent +): Promise { + await fetch({ + url: new URL(`${getRealtimeURL().toString()}api/runtime/events/user`), + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: token.asString(), + }, + body: JSON.stringify(event), + timeoutInMS: 5000, + }); +} From 629a1a8d5bdca228608e293214b7caf466fd27ba Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Sun, 5 Apr 2026 18:42:31 +0200 Subject: [PATCH 04/47] Fix user events endpoint and drop userAgent --- library/agent/api/UserEventsAPI.ts | 3 +-- library/agent/context/track.ts | 6 ------ 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/library/agent/api/UserEventsAPI.ts b/library/agent/api/UserEventsAPI.ts index 7dfa0c323..384d1c7ff 100644 --- a/library/agent/api/UserEventsAPI.ts +++ b/library/agent/api/UserEventsAPI.ts @@ -6,7 +6,6 @@ export type UserEvent = { name: string; userId: string | undefined; ipAddress: string | undefined; - userAgent: string | undefined; }; export async function sendUserEvent( @@ -14,7 +13,7 @@ export async function sendUserEvent( event: UserEvent ): Promise { await fetch({ - url: new URL(`${getRealtimeURL().toString()}api/runtime/events/user`), + url: new URL(`${getRealtimeURL().toString()}api/runtime/events`), method: "POST", headers: { "Content-Type": "application/json", diff --git a/library/agent/context/track.ts b/library/agent/context/track.ts index 312c6e850..862d14a75 100644 --- a/library/agent/context/track.ts +++ b/library/agent/context/track.ts @@ -19,16 +19,10 @@ export function track(eventName: string): void { return; } - const userAgent = - typeof context.headers["user-agent"] === "string" - ? context.headers["user-agent"] - : undefined; - agent.onTrackEvent({ name: eventName, userId: context.user?.id, ipAddress: context.remoteAddress, - userAgent, }); } From 994a8020abdf4774d437d15a11478c5c1c7a2d60 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Mon, 6 Apr 2026 11:33:37 +0200 Subject: [PATCH 05/47] Replace config polling with SSE streaming Use Server-Sent Events to receive near-instant config updates from zen-realtime instead of polling every 60 seconds. Falls back to polling when the SSE connection is unavailable. Vendors the eventsource-parser library (MIT) for SSE protocol parsing. --- library/agent/Agent.ts | 9 +- library/agent/realtime/connectToSSE.ts | 153 +++++++++++++ .../agent/realtime/listenForConfigUpdates.ts | 110 +++++++++ library/agent/realtime/pollForChanges.test.ts | 216 ------------------ library/agent/realtime/pollForChanges.ts | 54 ----- library/helpers/eventsource-parser/errors.ts | 23 ++ library/helpers/eventsource-parser/parse.ts | 170 ++++++++++++++ library/helpers/eventsource-parser/types.ts | 22 ++ 8 files changed, 483 insertions(+), 274 deletions(-) create mode 100644 library/agent/realtime/connectToSSE.ts create mode 100644 library/agent/realtime/listenForConfigUpdates.ts delete mode 100644 library/agent/realtime/pollForChanges.test.ts delete mode 100644 library/agent/realtime/pollForChanges.ts create mode 100644 library/helpers/eventsource-parser/errors.ts create mode 100644 library/helpers/eventsource-parser/parse.ts create mode 100644 library/helpers/eventsource-parser/types.ts diff --git a/library/agent/Agent.ts b/library/agent/Agent.ts index f9dda705e..3af55b63e 100644 --- a/library/agent/Agent.ts +++ b/library/agent/Agent.ts @@ -17,7 +17,7 @@ import { sendUserEvent, type UserEvent } from "./api/UserEventsAPI"; import { Token } from "./api/Token"; import { Kind } from "./Attack"; import { Endpoint } from "./Config"; -import { pollForChanges } from "./realtime/pollForChanges"; +import { listenForConfigUpdates } from "./realtime/listenForConfigUpdates"; import { Context } from "./Context"; import { Hostnames } from "./Hostnames"; import { InspectionStatistics } from "./InspectionStatistics"; @@ -67,6 +67,7 @@ export class Agent { private attackLogger = new AttackLogger(1000); private attackWaveDetector = new AttackWaveDetector(); private pendingEvents = new PendingEvents(); + private configListener: { stop(): void } | undefined = undefined; private idorProtectionConfig: IdorProtectionConfig | undefined = undefined; constructor( @@ -443,8 +444,8 @@ export class Agent { } } - private startPollingForConfigChanges() { - pollForChanges({ + private startListeningForConfigUpdates() { + this.configListener = listenForConfigUpdates({ token: this.token, logger: this.logger, lastUpdatedAt: this.serviceConfig.getLastUpdatedAt(), @@ -540,7 +541,7 @@ export class Agent { this.onStart() .then(() => { this.startHeartbeats(); - this.startPollingForConfigChanges(); + this.startListeningForConfigUpdates(); }) .catch((err) => { console.error(`Aikido: Failed to start agent: ${err.message}`); diff --git a/library/agent/realtime/connectToSSE.ts b/library/agent/realtime/connectToSSE.ts new file mode 100644 index 000000000..99fdc2089 --- /dev/null +++ b/library/agent/realtime/connectToSSE.ts @@ -0,0 +1,153 @@ +import { request as requestHttp } from "http"; +import { request as requestHttps } from "https"; +import { createParser } from "../../helpers/eventsource-parser/parse"; +import type { EventSourceMessage } from "../../helpers/eventsource-parser/types"; +import { Token } from "../api/Token"; +import { Logger } from "../logger/Logger"; +import { getRealtimeURL } from "./getRealtimeURL"; + +const INITIAL_RECONNECT_MS = 1000; +const MAX_RECONNECT_MS = 60 * 1000; + +type SSEConnection = { + close(): void; +}; + +export function connectToSSE({ + token, + logger, + onEvent, + onConnect, + onDisconnect, +}: { + token: Token; + logger: Logger; + onEvent: (event: EventSourceMessage) => void; + onConnect?: () => void; + onDisconnect?: () => void; +}): SSEConnection { + let closed = false; + let reconnectMs = INITIAL_RECONNECT_MS; + let reconnectTimer: NodeJS.Timeout | null = null; + let currentRequest: ReturnType | null = null; + + function connect() { + if (closed) { + return; + } + + const url = new URL( + `${getRealtimeURL().toString()}api/runtime/stream` + ); + + const requestFn = url.protocol === "https:" ? requestHttps : requestHttp; + + const req = requestFn( + url.toString(), + { + method: "GET", + headers: { + Authorization: token.asString(), + Accept: "text/event-stream", + "Cache-Control": "no-cache", + }, + }, + (response) => { + if (closed) { + response.destroy(); + return; + } + + if (response.statusCode !== 200) { + logger.log( + `SSE connection failed with status ${response.statusCode}` + ); + response.destroy(); + scheduleReconnect(); + return; + } + + // Successfully connected, reset backoff + reconnectMs = INITIAL_RECONNECT_MS; + if (onConnect) { + onConnect(); + } + + const parser = createParser({ + onEvent(event) { + onEvent(event); + }, + }); + + response.setEncoding("utf-8"); + + response.on("data", (chunk: string) => { + parser.feed(chunk); + }); + + response.on("end", () => { + if (!closed) { + logger.log("SSE connection closed by server, reconnecting"); + parser.reset(); + scheduleReconnect(); + } + }); + + response.on("error", (error) => { + if (!closed) { + logger.log(`SSE stream error: ${error.message}`); + parser.reset(); + scheduleReconnect(); + } + }); + } + ); + + currentRequest = req; + + req.on("error", (error) => { + if (!closed) { + logger.log(`SSE connection error: ${error.message}`); + scheduleReconnect(); + } + }); + + req.end(); + } + + function scheduleReconnect() { + if (closed) { + return; + } + + if (onDisconnect) { + onDisconnect(); + } + + // Exponential backoff with jitter + const jitter = Math.random() * 0.5 + 0.75; // 0.75 - 1.25 + const delay = Math.min(reconnectMs * jitter, MAX_RECONNECT_MS); + reconnectMs = Math.min(reconnectMs * 2, MAX_RECONNECT_MS); + + reconnectTimer = setTimeout(connect, delay); + reconnectTimer.unref(); + } + + function close() { + closed = true; + + if (reconnectTimer) { + clearTimeout(reconnectTimer); + reconnectTimer = null; + } + + if (currentRequest) { + currentRequest.destroy(); + currentRequest = null; + } + } + + connect(); + + return { close }; +} diff --git a/library/agent/realtime/listenForConfigUpdates.ts b/library/agent/realtime/listenForConfigUpdates.ts new file mode 100644 index 000000000..d156361ab --- /dev/null +++ b/library/agent/realtime/listenForConfigUpdates.ts @@ -0,0 +1,110 @@ +import { Token } from "../api/Token"; +import { Config } from "../Config"; +import { Logger } from "../logger/Logger"; +import { connectToSSE } from "./connectToSSE"; +import { getConfig } from "./getConfig"; +import { getConfigLastUpdatedAt } from "./getConfigLastUpdatedAt"; + +type OnConfigUpdate = (config: Config) => void; + +type ConfigListener = { + stop(): void; +}; + +export function listenForConfigUpdates({ + onConfigUpdate, + token, + logger, + lastUpdatedAt, +}: { + onConfigUpdate: OnConfigUpdate; + token: Token | undefined; + logger: Logger; + lastUpdatedAt: number; +}): ConfigListener { + if (!token) { + logger.log("No token provided, not listening for config updates"); + return { stop() {} }; + } + + let currentLastUpdatedAt = lastUpdatedAt; + let pollingInterval: NodeJS.Timeout | null = null; + + function startPolling() { + if (pollingInterval) { + return; + } + + pollingInterval = setInterval(() => { + checkForUpdates().catch((error) => { + logger.log(`Failed to check for config updates: ${error.message}`); + }); + }, 60 * 1000); + + pollingInterval.unref(); + } + + function stopPolling() { + if (pollingInterval) { + clearInterval(pollingInterval); + pollingInterval = null; + } + } + + async function checkForUpdates() { + const configLastUpdatedAt = await getConfigLastUpdatedAt(token!); + + if (configLastUpdatedAt > currentLastUpdatedAt) { + const config = await getConfig(token!); + currentLastUpdatedAt = config.configUpdatedAt; + onConfigUpdate(config); + } + } + + // Start SSE connection for near-instant config updates + const sse = connectToSSE({ + token, + logger, + onConnect() { + stopPolling(); + }, + onDisconnect() { + startPolling(); + }, + onEvent(event) { + if (event.event !== "config-updated") { + return; + } + + try { + const payload: { configUpdatedAt: number } = JSON.parse(event.data); + if (payload.configUpdatedAt <= currentLastUpdatedAt) { + return; + } + } catch { + // If we can't parse the payload, fetch the config anyway + } + + getConfig(token!) + .then((config) => { + currentLastUpdatedAt = config.configUpdatedAt; + onConfigUpdate(config); + }) + .catch((error) => { + logger.log( + `Failed to fetch config after SSE event: ${error.message}` + ); + }); + }, + }); + + // Start polling as fallback until SSE connects + startPolling(); + + return { + stop() { + sse.close(); + stopPolling(); + }, + }; +} diff --git a/library/agent/realtime/pollForChanges.test.ts b/library/agent/realtime/pollForChanges.test.ts deleted file mode 100644 index abfe9be68..000000000 --- a/library/agent/realtime/pollForChanges.test.ts +++ /dev/null @@ -1,216 +0,0 @@ -import * as t from "tap"; -import * as fetch from "../../helpers/fetch"; -import { wrap } from "../../helpers/wrap"; -import { Token } from "../api/Token"; -import { LoggerForTesting } from "../logger/LoggerForTesting"; -import { LoggerNoop } from "../logger/LoggerNoop"; -import { pollForChanges } from "./pollForChanges"; -import * as FakeTimers from "@sinonjs/fake-timers"; -import { Config } from "../Config"; - -t.test("it does not start interval if no token", async (t) => { - const logger = new LoggerForTesting(); - pollForChanges({ - onConfigUpdate: (config) => t.fail(), - logger: logger, - token: undefined, - lastUpdatedAt: 0, - }); - - t.same(logger.getMessages(), [ - "No token provided, not polling for config updates", - ]); -}); - -t.test("it checks for config updates", async () => { - const clock = FakeTimers.install(); - - const calls: { url: string; method: string }[] = []; - let configUpdatedAt = 0; - - wrap(fetch, "fetch", function fetch() { - return async function fetch(params: any) { - calls.push({ - url: params.url.toString(), - method: params.method, - }); - - if (params.url.hostname.startsWith("runtime")) { - return { - body: JSON.stringify({ - configUpdatedAt: configUpdatedAt, - }), - statusCode: 200, - }; - } - - if (params.url.hostname.startsWith("guard")) { - return { - body: JSON.stringify({ - endpoints: [], - heartbeatIntervalInMS: 10 * 60 * 1000, - configUpdatedAt: configUpdatedAt, - }), - statusCode: 200, - }; - } - - throw new Error(`Unknown hostname: ${params.url.hostname}`); - }; - }); - - const configUpdates: Config[] = []; - - pollForChanges({ - onConfigUpdate: (config) => { - configUpdates.push(config); - }, - logger: new LoggerNoop(), - token: new Token("123"), - lastUpdatedAt: 0, - }); - - t.same(configUpdates, []); - t.same(calls, []); - - await clock.nextAsync(); - - t.same(configUpdates, []); - t.same(calls, [ - { - url: "https://runtime.aikido.dev/config", - method: "GET", - }, - ]); - - configUpdatedAt = 1; - await clock.nextAsync(); - - t.same(configUpdates, [ - { - endpoints: [], - heartbeatIntervalInMS: 10 * 60 * 1000, - configUpdatedAt: 1, - }, - ]); - t.same(calls, [ - { - url: "https://runtime.aikido.dev/config", - method: "GET", - }, - { - url: "https://runtime.aikido.dev/config", - method: "GET", - }, - { - url: "https://guard.aikido.dev/api/runtime/config", - method: "GET", - }, - ]); - - await clock.nextAsync(); - - t.same(configUpdates, [ - { - endpoints: [], - heartbeatIntervalInMS: 10 * 60 * 1000, - configUpdatedAt: 1, - }, - ]); - t.same(calls, [ - { - url: "https://runtime.aikido.dev/config", - method: "GET", - }, - { - url: "https://runtime.aikido.dev/config", - method: "GET", - }, - { - url: "https://guard.aikido.dev/api/runtime/config", - method: "GET", - }, - { - url: "https://runtime.aikido.dev/config", - method: "GET", - }, - ]); - - configUpdatedAt = 2; - await clock.nextAsync(); - - t.same(configUpdates, [ - { - endpoints: [], - heartbeatIntervalInMS: 10 * 60 * 1000, - configUpdatedAt: 1, - }, - { - endpoints: [], - heartbeatIntervalInMS: 10 * 60 * 1000, - configUpdatedAt: 2, - }, - ]); - t.same(calls, [ - { - url: "https://runtime.aikido.dev/config", - method: "GET", - }, - { - url: "https://runtime.aikido.dev/config", - method: "GET", - }, - { - url: "https://guard.aikido.dev/api/runtime/config", - method: "GET", - }, - { - url: "https://runtime.aikido.dev/config", - method: "GET", - }, - { - url: "https://runtime.aikido.dev/config", - method: "GET", - }, - { - url: "https://guard.aikido.dev/api/runtime/config", - method: "GET", - }, - ]); - - clock.uninstall(); -}); - -t.test("it deals with API throwing errors", async () => { - const clock = FakeTimers.install(); - - wrap(fetch, "fetch", function fetch() { - return async function fetch() { - throw new Error("Request timed out"); - }; - }); - - const configUpdates: Config[] = []; - - const logger = new LoggerForTesting(); - pollForChanges({ - onConfigUpdate: (config) => { - configUpdates.push(config); - }, - logger: logger, - token: new Token("123"), - lastUpdatedAt: 0, - }); - - t.same(configUpdates, []); - t.same(logger.getMessages(), []); - - await clock.nextAsync(); - - t.same(configUpdates, []); - t.same(logger.getMessages(), [ - `Failed to check for config updates: Request timed out`, - ]); - - clock.uninstall(); -}); diff --git a/library/agent/realtime/pollForChanges.ts b/library/agent/realtime/pollForChanges.ts deleted file mode 100644 index 25825d410..000000000 --- a/library/agent/realtime/pollForChanges.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { Token } from "../api/Token"; -import { Config } from "../Config"; -import { Logger } from "../logger/Logger"; -import { getConfig } from "./getConfig"; -import { getConfigLastUpdatedAt } from "./getConfigLastUpdatedAt"; - -type OnConfigUpdate = (config: Config) => void; - -let interval: NodeJS.Timeout | null = null; -let currentLastUpdatedAt: number | null = null; - -export function pollForChanges({ - onConfigUpdate, - token, - logger, - lastUpdatedAt, -}: { - onConfigUpdate: OnConfigUpdate; - token: Token | undefined; - logger: Logger; - lastUpdatedAt: number; -}) { - if (!token) { - logger.log("No token provided, not polling for config updates"); - return; - } - - currentLastUpdatedAt = lastUpdatedAt; - - if (interval) { - clearInterval(interval); - } - - interval = setInterval(() => { - check(token, onConfigUpdate).catch((error) => { - logger.log(`Failed to check for config updates: ${error.message}`); - }); - }, 60 * 1000); - - interval.unref(); -} - -async function check(token: Token, onConfigUpdate: OnConfigUpdate) { - const configLastUpdatedAt = await getConfigLastUpdatedAt(token); - - if ( - typeof currentLastUpdatedAt === "number" && - configLastUpdatedAt > currentLastUpdatedAt - ) { - const config = await getConfig(token); - currentLastUpdatedAt = config.configUpdatedAt; - onConfigUpdate(config); - } -} diff --git a/library/helpers/eventsource-parser/errors.ts b/library/helpers/eventsource-parser/errors.ts new file mode 100644 index 000000000..d34b2f29e --- /dev/null +++ b/library/helpers/eventsource-parser/errors.ts @@ -0,0 +1,23 @@ +// Based on https://github.com/rexxars/eventsource-parser +// MIT License - Copyright (c) 2025 Espen Hovlandsdal + +export type ErrorType = "invalid-retry" | "unknown-field"; + +export class ParseError extends Error { + type: ErrorType; + field?: string | undefined; + value?: string | undefined; + line?: string | undefined; + + constructor( + message: string, + options: { type: ErrorType; field?: string; value?: string; line?: string } + ) { + super(message); + this.name = "ParseError"; + this.type = options.type; + this.field = options.field; + this.value = options.value; + this.line = options.line; + } +} diff --git a/library/helpers/eventsource-parser/parse.ts b/library/helpers/eventsource-parser/parse.ts new file mode 100644 index 000000000..780320021 --- /dev/null +++ b/library/helpers/eventsource-parser/parse.ts @@ -0,0 +1,170 @@ +// Based on https://github.com/rexxars/eventsource-parser +// MIT License - Copyright (c) 2025 Espen Hovlandsdal + +import { ParseError } from "./errors"; +import type { EventSourceParser, ParserCallbacks } from "./types"; + +function noop(_arg: unknown) { + // intentional noop +} + +export function createParser(callbacks: ParserCallbacks): EventSourceParser { + const { + onEvent = noop, + onError = noop, + onRetry = noop, + onComment, + } = callbacks; + + let incompleteLine = ""; + let isFirstChunk = true; + let id: string | undefined; + let data = ""; + let eventType = ""; + + function feed(newChunk: string) { + // Strip any UTF8 byte order mark (BOM) at the start of the stream + const chunk = isFirstChunk + ? newChunk.replace(/^\xEF\xBB\xBF/, "") + : newChunk; + + const [complete, incomplete] = splitLines(`${incompleteLine}${chunk}`); + + for (const line of complete) { + parseLine(line); + } + + incompleteLine = incomplete; + isFirstChunk = false; + } + + function parseLine(line: string) { + if (line === "") { + dispatchEvent(); + return; + } + + if (line.startsWith(":")) { + if (onComment) { + onComment(line.slice(line.startsWith(": ") ? 2 : 1)); + } + return; + } + + const fieldSeparatorIndex = line.indexOf(":"); + if (fieldSeparatorIndex !== -1) { + const field = line.slice(0, fieldSeparatorIndex); + const offset = line[fieldSeparatorIndex + 1] === " " ? 2 : 1; + const value = line.slice(fieldSeparatorIndex + offset); + processField(field, value, line); + return; + } + + processField(line, "", line); + } + + function processField(field: string, value: string, line: string) { + switch (field) { + case "event": + eventType = value; + break; + case "data": + data = `${data}${value}\n`; + break; + case "id": + id = value.includes("\0") ? undefined : value; + break; + case "retry": + if (/^\d+$/.test(value)) { + onRetry(parseInt(value, 10)); + } else { + onError( + new ParseError(`Invalid \`retry\` value: "${value}"`, { + type: "invalid-retry", + value, + line, + }) + ); + } + break; + default: + onError( + new ParseError( + `Unknown field "${field.length > 20 ? `${field.slice(0, 20)}…` : field}"`, + { type: "unknown-field", field, value, line } + ) + ); + break; + } + } + + function dispatchEvent() { + const shouldDispatch = data.length > 0; + if (shouldDispatch) { + onEvent({ + id, + event: eventType || undefined, + data: data.endsWith("\n") ? data.slice(0, -1) : data, + }); + } + + id = undefined; + data = ""; + eventType = ""; + } + + function reset(options: { consume?: boolean } = {}) { + if (incompleteLine && options.consume) { + parseLine(incompleteLine); + } + + isFirstChunk = true; + id = undefined; + data = ""; + eventType = ""; + incompleteLine = ""; + } + + return { feed, reset }; +} + +function splitLines( + chunk: string +): [complete: Array, incomplete: string] { + const lines: Array = []; + let incompleteLine = ""; + let searchIndex = 0; + + while (searchIndex < chunk.length) { + const crIndex = chunk.indexOf("\r", searchIndex); + const lfIndex = chunk.indexOf("\n", searchIndex); + + let lineEnd = -1; + if (crIndex !== -1 && lfIndex !== -1) { + lineEnd = Math.min(crIndex, lfIndex); + } else if (crIndex !== -1) { + if (crIndex === chunk.length - 1) { + lineEnd = -1; + } else { + lineEnd = crIndex; + } + } else if (lfIndex !== -1) { + lineEnd = lfIndex; + } + + if (lineEnd === -1) { + incompleteLine = chunk.slice(searchIndex); + break; + } else { + const line = chunk.slice(searchIndex, lineEnd); + lines.push(line); + + searchIndex = lineEnd + 1; + if (chunk[searchIndex - 1] === "\r" && chunk[searchIndex] === "\n") { + searchIndex++; + } + } + } + + return [lines, incompleteLine]; +} diff --git a/library/helpers/eventsource-parser/types.ts b/library/helpers/eventsource-parser/types.ts new file mode 100644 index 000000000..cf8130d56 --- /dev/null +++ b/library/helpers/eventsource-parser/types.ts @@ -0,0 +1,22 @@ +// Based on https://github.com/rexxars/eventsource-parser +// MIT License - Copyright (c) 2025 Espen Hovlandsdal + +import type { ParseError } from "./errors"; + +export interface EventSourceParser { + feed(chunk: string): void; + reset(options?: { consume?: boolean }): void; +} + +export interface EventSourceMessage { + event?: string | undefined; + id?: string | undefined; + data: string; +} + +export interface ParserCallbacks { + onEvent?: ((event: EventSourceMessage) => void) | undefined; + onRetry?: ((retry: number) => void) | undefined; + onComment?: ((comment: string) => void) | undefined; + onError?: ((error: ParseError) => void) | undefined; +} From abbf9b66b18e7c32d8568834dba25d7937abb89c Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Mon, 6 Apr 2026 11:46:54 +0200 Subject: [PATCH 06/47] Simplify SSE connection and remove unused cleanup --- library/agent/Agent.ts | 3 +- library/agent/realtime/connectToSSE.ts | 68 +++++-------------- .../agent/realtime/listenForConfigUpdates.ts | 26 ++----- 3 files changed, 25 insertions(+), 72 deletions(-) diff --git a/library/agent/Agent.ts b/library/agent/Agent.ts index 3af55b63e..9556e9b84 100644 --- a/library/agent/Agent.ts +++ b/library/agent/Agent.ts @@ -67,7 +67,6 @@ export class Agent { private attackLogger = new AttackLogger(1000); private attackWaveDetector = new AttackWaveDetector(); private pendingEvents = new PendingEvents(); - private configListener: { stop(): void } | undefined = undefined; private idorProtectionConfig: IdorProtectionConfig | undefined = undefined; constructor( @@ -445,7 +444,7 @@ export class Agent { } private startListeningForConfigUpdates() { - this.configListener = listenForConfigUpdates({ + listenForConfigUpdates({ token: this.token, logger: this.logger, lastUpdatedAt: this.serviceConfig.getLastUpdatedAt(), diff --git a/library/agent/realtime/connectToSSE.ts b/library/agent/realtime/connectToSSE.ts index 99fdc2089..a5a0d2849 100644 --- a/library/agent/realtime/connectToSSE.ts +++ b/library/agent/realtime/connectToSSE.ts @@ -9,10 +9,6 @@ import { getRealtimeURL } from "./getRealtimeURL"; const INITIAL_RECONNECT_MS = 1000; const MAX_RECONNECT_MS = 60 * 1000; -type SSEConnection = { - close(): void; -}; - export function connectToSSE({ token, logger, @@ -25,20 +21,18 @@ export function connectToSSE({ onEvent: (event: EventSourceMessage) => void; onConnect?: () => void; onDisconnect?: () => void; -}): SSEConnection { - let closed = false; +}) { let reconnectMs = INITIAL_RECONNECT_MS; let reconnectTimer: NodeJS.Timeout | null = null; let currentRequest: ReturnType | null = null; function connect() { - if (closed) { - return; + if (currentRequest) { + currentRequest.destroy(); + currentRequest = null; } - const url = new URL( - `${getRealtimeURL().toString()}api/runtime/stream` - ); + const url = new URL(`${getRealtimeURL().toString()}api/runtime/stream`); const requestFn = url.protocol === "https:" ? requestHttps : requestHttp; @@ -53,11 +47,6 @@ export function connectToSSE({ }, }, (response) => { - if (closed) { - response.destroy(); - return; - } - if (response.statusCode !== 200) { logger.log( `SSE connection failed with status ${response.statusCode}` @@ -67,7 +56,6 @@ export function connectToSSE({ return; } - // Successfully connected, reset backoff reconnectMs = INITIAL_RECONNECT_MS; if (onConnect) { onConnect(); @@ -86,40 +74,34 @@ export function connectToSSE({ }); response.on("end", () => { - if (!closed) { - logger.log("SSE connection closed by server, reconnecting"); - parser.reset(); - scheduleReconnect(); - } + logger.log("SSE connection closed by server, reconnecting"); + parser.reset(); + scheduleReconnect(); }); response.on("error", (error) => { - if (!closed) { - logger.log(`SSE stream error: ${error.message}`); - parser.reset(); - scheduleReconnect(); - } + logger.log(`SSE stream error: ${error.message}`); + parser.reset(); + scheduleReconnect(); }); } ); currentRequest = req; + req.on("socket", (socket) => { + socket.unref(); + }); + req.on("error", (error) => { - if (!closed) { - logger.log(`SSE connection error: ${error.message}`); - scheduleReconnect(); - } + logger.log(`SSE connection error: ${error.message}`); + scheduleReconnect(); }); req.end(); } function scheduleReconnect() { - if (closed) { - return; - } - if (onDisconnect) { onDisconnect(); } @@ -133,21 +115,5 @@ export function connectToSSE({ reconnectTimer.unref(); } - function close() { - closed = true; - - if (reconnectTimer) { - clearTimeout(reconnectTimer); - reconnectTimer = null; - } - - if (currentRequest) { - currentRequest.destroy(); - currentRequest = null; - } - } - connect(); - - return { close }; } diff --git a/library/agent/realtime/listenForConfigUpdates.ts b/library/agent/realtime/listenForConfigUpdates.ts index d156361ab..53db64f9b 100644 --- a/library/agent/realtime/listenForConfigUpdates.ts +++ b/library/agent/realtime/listenForConfigUpdates.ts @@ -7,10 +7,6 @@ import { getConfigLastUpdatedAt } from "./getConfigLastUpdatedAt"; type OnConfigUpdate = (config: Config) => void; -type ConfigListener = { - stop(): void; -}; - export function listenForConfigUpdates({ onConfigUpdate, token, @@ -21,12 +17,13 @@ export function listenForConfigUpdates({ token: Token | undefined; logger: Logger; lastUpdatedAt: number; -}): ConfigListener { +}) { if (!token) { logger.log("No token provided, not listening for config updates"); - return { stop() {} }; + return; } + const validToken = token; let currentLastUpdatedAt = lastUpdatedAt; let pollingInterval: NodeJS.Timeout | null = null; @@ -52,17 +49,16 @@ export function listenForConfigUpdates({ } async function checkForUpdates() { - const configLastUpdatedAt = await getConfigLastUpdatedAt(token!); + const configLastUpdatedAt = await getConfigLastUpdatedAt(validToken); if (configLastUpdatedAt > currentLastUpdatedAt) { - const config = await getConfig(token!); + const config = await getConfig(validToken); currentLastUpdatedAt = config.configUpdatedAt; onConfigUpdate(config); } } - // Start SSE connection for near-instant config updates - const sse = connectToSSE({ + connectToSSE({ token, logger, onConnect() { @@ -85,7 +81,7 @@ export function listenForConfigUpdates({ // If we can't parse the payload, fetch the config anyway } - getConfig(token!) + getConfig(validToken) .then((config) => { currentLastUpdatedAt = config.configUpdatedAt; onConfigUpdate(config); @@ -98,13 +94,5 @@ export function listenForConfigUpdates({ }, }); - // Start polling as fallback until SSE connects startPolling(); - - return { - stop() { - sse.close(); - stopPolling(); - }, - }; } From 34984545556c7e3eb734fb5275eb302d942c367c Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Mon, 6 Apr 2026 11:55:20 +0200 Subject: [PATCH 07/47] Add login route with track events to express-mysql sample app --- sample-apps/express-mysql/app.js | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/sample-apps/express-mysql/app.js b/sample-apps/express-mysql/app.js index 7f549eec5..d7306affe 100644 --- a/sample-apps/express-mysql/app.js +++ b/sample-apps/express-mysql/app.js @@ -1,5 +1,5 @@ require("dotenv").config(); -require("@aikidosec/firewall"); +const { track, setUser } = require("@aikidosec/firewall"); const Sentry = require("@sentry/node"); Sentry.init({ @@ -108,6 +108,30 @@ async function main(port) { res.status(200).send("Done"); }); + app.post( + "/login", + express.json(), + asyncHandler(async (req, res) => { + const { username, password } = req.body; + + if (!username || !password) { + return res + .status(400) + .json({ error: "username and password required" }); + } + + // Hardcoded credentials for demo purposes + if (username === "admin" && password === "admin") { + setUser({ id: "admin", name: "Admin" }); + track("login_success"); + return res.json({ message: "Login successful" }); + } + + track("login_failure"); + res.status(401).json({ error: "Invalid credentials" }); + }) + ); + // This route is for testing purposes only and uses internal APIs // Normal users should NOT rely on these internals as they may change without notice app.get("/pending-events", (req, res) => { From 6084a60fde5ffcfda99982d65a5fb9accd1d8ad5 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Mon, 6 Apr 2026 19:37:34 +0200 Subject: [PATCH 08/47] Add AIKIDO_DEBUG_SSE env var and remove polling fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SSE is now the only mechanism for config updates — no more polling. Verbose SSE logging (connect, disconnect, chunks, events) is gated behind AIKIDO_DEBUG_SSE=true. Removes unused getConfigLastUpdatedAt. --- library/agent/realtime/connectToSSE.ts | 26 +++++---- .../agent/realtime/getConfigLastUpdatedAt.ts | 28 ---------- .../agent/realtime/listenForConfigUpdates.ts | 55 +++++-------------- library/helpers/isDebuggingSSE.ts | 5 ++ 4 files changed, 35 insertions(+), 79 deletions(-) delete mode 100644 library/agent/realtime/getConfigLastUpdatedAt.ts create mode 100644 library/helpers/isDebuggingSSE.ts diff --git a/library/agent/realtime/connectToSSE.ts b/library/agent/realtime/connectToSSE.ts index a5a0d2849..116a36b1c 100644 --- a/library/agent/realtime/connectToSSE.ts +++ b/library/agent/realtime/connectToSSE.ts @@ -2,6 +2,7 @@ import { request as requestHttp } from "http"; import { request as requestHttps } from "https"; import { createParser } from "../../helpers/eventsource-parser/parse"; import type { EventSourceMessage } from "../../helpers/eventsource-parser/types"; +import { isDebuggingSSE } from "../../helpers/isDebuggingSSE"; import { Token } from "../api/Token"; import { Logger } from "../logger/Logger"; import { getRealtimeURL } from "./getRealtimeURL"; @@ -13,19 +14,17 @@ export function connectToSSE({ token, logger, onEvent, - onConnect, - onDisconnect, }: { token: Token; logger: Logger; onEvent: (event: EventSourceMessage) => void; - onConnect?: () => void; - onDisconnect?: () => void; }) { let reconnectMs = INITIAL_RECONNECT_MS; let reconnectTimer: NodeJS.Timeout | null = null; let currentRequest: ReturnType | null = null; + const debugSSE = isDebuggingSSE(); + function connect() { if (currentRequest) { currentRequest.destroy(); @@ -34,6 +33,10 @@ export function connectToSSE({ const url = new URL(`${getRealtimeURL().toString()}api/runtime/stream`); + if (debugSSE) { + logger.log(`SSE connecting to ${url.toString()}`); + } + const requestFn = url.protocol === "https:" ? requestHttps : requestHttp; const req = requestFn( @@ -57,8 +60,8 @@ export function connectToSSE({ } reconnectMs = INITIAL_RECONNECT_MS; - if (onConnect) { - onConnect(); + if (debugSSE) { + logger.log("SSE connected successfully"); } const parser = createParser({ @@ -70,6 +73,9 @@ export function connectToSSE({ response.setEncoding("utf-8"); response.on("data", (chunk: string) => { + if (debugSSE) { + logger.log(`SSE received chunk: ${chunk.trimEnd()}`); + } parser.feed(chunk); }); @@ -102,15 +108,15 @@ export function connectToSSE({ } function scheduleReconnect() { - if (onDisconnect) { - onDisconnect(); - } - // Exponential backoff with jitter const jitter = Math.random() * 0.5 + 0.75; // 0.75 - 1.25 const delay = Math.min(reconnectMs * jitter, MAX_RECONNECT_MS); reconnectMs = Math.min(reconnectMs * 2, MAX_RECONNECT_MS); + if (debugSSE) { + logger.log(`SSE scheduling reconnect in ${Math.round(delay)}ms`); + } + reconnectTimer = setTimeout(connect, delay); reconnectTimer.unref(); } diff --git a/library/agent/realtime/getConfigLastUpdatedAt.ts b/library/agent/realtime/getConfigLastUpdatedAt.ts deleted file mode 100644 index 28f6512fa..000000000 --- a/library/agent/realtime/getConfigLastUpdatedAt.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { fetch } from "../../helpers/fetch"; -import { Token } from "../api/Token"; -import { getRealtimeURL } from "./getRealtimeURL"; - -type RealtimeResponse = { configUpdatedAt: number }; - -export async function getConfigLastUpdatedAt(token: Token): Promise { - const { body, statusCode } = await fetch({ - url: new URL(`${getRealtimeURL().toString()}config`), - method: "GET", - headers: { - Authorization: token.asString(), - }, - timeoutInMS: 3000, - }); - - if (statusCode === 401) { - throw new Error("Token is invalid"); - } - - if (statusCode !== 200) { - throw new Error(`Expected status code 200, got ${statusCode}`); - } - - const response: RealtimeResponse = JSON.parse(body); - - return response.configUpdatedAt; -} diff --git a/library/agent/realtime/listenForConfigUpdates.ts b/library/agent/realtime/listenForConfigUpdates.ts index 53db64f9b..e48e5d73d 100644 --- a/library/agent/realtime/listenForConfigUpdates.ts +++ b/library/agent/realtime/listenForConfigUpdates.ts @@ -1,9 +1,9 @@ +import { isDebuggingSSE } from "../../helpers/isDebuggingSSE"; import { Token } from "../api/Token"; import { Config } from "../Config"; import { Logger } from "../logger/Logger"; import { connectToSSE } from "./connectToSSE"; import { getConfig } from "./getConfig"; -import { getConfigLastUpdatedAt } from "./getConfigLastUpdatedAt"; type OnConfigUpdate = (config: Config) => void; @@ -24,50 +24,16 @@ export function listenForConfigUpdates({ } const validToken = token; + const debugSSE = isDebuggingSSE(); let currentLastUpdatedAt = lastUpdatedAt; - let pollingInterval: NodeJS.Timeout | null = null; - - function startPolling() { - if (pollingInterval) { - return; - } - - pollingInterval = setInterval(() => { - checkForUpdates().catch((error) => { - logger.log(`Failed to check for config updates: ${error.message}`); - }); - }, 60 * 1000); - - pollingInterval.unref(); - } - - function stopPolling() { - if (pollingInterval) { - clearInterval(pollingInterval); - pollingInterval = null; - } - } - - async function checkForUpdates() { - const configLastUpdatedAt = await getConfigLastUpdatedAt(validToken); - - if (configLastUpdatedAt > currentLastUpdatedAt) { - const config = await getConfig(validToken); - currentLastUpdatedAt = config.configUpdatedAt; - onConfigUpdate(config); - } - } connectToSSE({ token, logger, - onConnect() { - stopPolling(); - }, - onDisconnect() { - startPolling(); - }, onEvent(event) { + if (debugSSE) { + logger.log(`SSE event received: ${event.event}`); + } if (event.event !== "config-updated") { return; } @@ -81,8 +47,17 @@ export function listenForConfigUpdates({ // If we can't parse the payload, fetch the config anyway } + if (debugSSE) { + logger.log("SSE config-updated event, fetching new config"); + } + getConfig(validToken) .then((config) => { + if (debugSSE) { + logger.log( + `SSE config fetched, configUpdatedAt: ${config.configUpdatedAt}` + ); + } currentLastUpdatedAt = config.configUpdatedAt; onConfigUpdate(config); }) @@ -93,6 +68,4 @@ export function listenForConfigUpdates({ }); }, }); - - startPolling(); } diff --git a/library/helpers/isDebuggingSSE.ts b/library/helpers/isDebuggingSSE.ts new file mode 100644 index 000000000..9282e532b --- /dev/null +++ b/library/helpers/isDebuggingSSE.ts @@ -0,0 +1,5 @@ +import { envToBool } from "./envToBool"; + +export function isDebuggingSSE() { + return envToBool(process.env.AIKIDO_DEBUG_SSE); +} From e91caf291da9e9ec58be499919fae44c44eb635c Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Mon, 6 Apr 2026 19:43:05 +0200 Subject: [PATCH 09/47] Clear existing timer before scheduling SSE reconnect --- library/agent/realtime/connectToSSE.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/library/agent/realtime/connectToSSE.ts b/library/agent/realtime/connectToSSE.ts index 116a36b1c..26396a8f6 100644 --- a/library/agent/realtime/connectToSSE.ts +++ b/library/agent/realtime/connectToSSE.ts @@ -108,6 +108,10 @@ export function connectToSSE({ } function scheduleReconnect() { + if (reconnectTimer) { + clearTimeout(reconnectTimer); + } + // Exponential backoff with jitter const jitter = Math.random() * 0.5 + 0.75; // 0.75 - 1.25 const delay = Math.min(reconnectMs * jitter, MAX_RECONNECT_MS); From 5f76bbb6ea07afdc2a205bb861c36eaa2c28a40f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6ssler?= Date: Fri, 24 Apr 2026 09:43:16 +0200 Subject: [PATCH 10/47] Add config updated polling again --- library/agent/Agent.ts | 13 +- library/agent/realtime/ConfigUpdateOptions.ts | 10 + .../agent/realtime/getConfigLastUpdatedAt.ts | 28 +++ .../agent/realtime/listenForConfigUpdates.ts | 13 +- library/agent/realtime/pollForChanges.test.ts | 216 ++++++++++++++++++ library/agent/realtime/pollForChanges.ts | 49 ++++ 6 files changed, 314 insertions(+), 15 deletions(-) create mode 100644 library/agent/realtime/ConfigUpdateOptions.ts create mode 100644 library/agent/realtime/getConfigLastUpdatedAt.ts create mode 100644 library/agent/realtime/pollForChanges.test.ts create mode 100644 library/agent/realtime/pollForChanges.ts diff --git a/library/agent/Agent.ts b/library/agent/Agent.ts index 7207dab09..66417ade7 100644 --- a/library/agent/Agent.ts +++ b/library/agent/Agent.ts @@ -18,6 +18,7 @@ import { Token } from "./api/Token"; import { Kind } from "./Attack"; import { Endpoint } from "./Config"; import { listenForConfigUpdates } from "./realtime/listenForConfigUpdates"; +import type { ConfigUpdateOptions } from "./realtime/ConfigUpdateOptions"; import { Context } from "./Context"; import { Hostnames } from "./Hostnames"; import { InspectionStatistics } from "./InspectionStatistics"; @@ -38,6 +39,7 @@ import type { FetchListsAPI } from "./api/FetchListsAPI"; import { PendingEvents } from "./PendingEvents"; import type { IdorProtectionConfig } from "./IdorProtectionConfig"; import { warnIfTsxIsUsed } from "../helpers/warnIfTsxIsUsed"; +import { pollForChanges } from "./realtime/pollForChanges"; type WrappedPackage = { version: string; supported: boolean }; @@ -452,8 +454,8 @@ export class Agent { } } - private startListeningForConfigUpdates() { - listenForConfigUpdates({ + private startCheckingForConfigUpdates() { + const options: ConfigUpdateOptions = { token: this.token, logger: this.logger, lastUpdatedAt: this.serviceConfig.getLastUpdatedAt(), @@ -463,7 +465,10 @@ export class Agent { this.logger.log(`Failed to update blocked lists: ${error.message}`); }); }, - }); + }; + + listenForConfigUpdates(options); + pollForChanges(options); } private getAgentInfo(): AgentInfo { @@ -549,7 +554,7 @@ export class Agent { this.onStart() .then(() => { this.startHeartbeats(); - this.startListeningForConfigUpdates(); + this.startCheckingForConfigUpdates(); }) .catch((err) => { console.error(`Aikido: Failed to start agent: ${err.message}`); diff --git a/library/agent/realtime/ConfigUpdateOptions.ts b/library/agent/realtime/ConfigUpdateOptions.ts new file mode 100644 index 000000000..2129c9833 --- /dev/null +++ b/library/agent/realtime/ConfigUpdateOptions.ts @@ -0,0 +1,10 @@ +import type { Token } from "../api/Token"; +import type { Config } from "../Config"; +import type { Logger } from "../logger/Logger"; + +export type ConfigUpdateOptions = { + onConfigUpdate: (config: Config) => void; + token: Token | undefined; + logger: Logger; + lastUpdatedAt: number; +}; diff --git a/library/agent/realtime/getConfigLastUpdatedAt.ts b/library/agent/realtime/getConfigLastUpdatedAt.ts new file mode 100644 index 000000000..28f6512fa --- /dev/null +++ b/library/agent/realtime/getConfigLastUpdatedAt.ts @@ -0,0 +1,28 @@ +import { fetch } from "../../helpers/fetch"; +import { Token } from "../api/Token"; +import { getRealtimeURL } from "./getRealtimeURL"; + +type RealtimeResponse = { configUpdatedAt: number }; + +export async function getConfigLastUpdatedAt(token: Token): Promise { + const { body, statusCode } = await fetch({ + url: new URL(`${getRealtimeURL().toString()}config`), + method: "GET", + headers: { + Authorization: token.asString(), + }, + timeoutInMS: 3000, + }); + + if (statusCode === 401) { + throw new Error("Token is invalid"); + } + + if (statusCode !== 200) { + throw new Error(`Expected status code 200, got ${statusCode}`); + } + + const response: RealtimeResponse = JSON.parse(body); + + return response.configUpdatedAt; +} diff --git a/library/agent/realtime/listenForConfigUpdates.ts b/library/agent/realtime/listenForConfigUpdates.ts index e48e5d73d..d493d3e99 100644 --- a/library/agent/realtime/listenForConfigUpdates.ts +++ b/library/agent/realtime/listenForConfigUpdates.ts @@ -1,23 +1,14 @@ import { isDebuggingSSE } from "../../helpers/isDebuggingSSE"; -import { Token } from "../api/Token"; -import { Config } from "../Config"; -import { Logger } from "../logger/Logger"; import { connectToSSE } from "./connectToSSE"; +import type { ConfigUpdateOptions } from "./ConfigUpdateOptions"; import { getConfig } from "./getConfig"; -type OnConfigUpdate = (config: Config) => void; - export function listenForConfigUpdates({ onConfigUpdate, token, logger, lastUpdatedAt, -}: { - onConfigUpdate: OnConfigUpdate; - token: Token | undefined; - logger: Logger; - lastUpdatedAt: number; -}) { +}: ConfigUpdateOptions) { if (!token) { logger.log("No token provided, not listening for config updates"); return; diff --git a/library/agent/realtime/pollForChanges.test.ts b/library/agent/realtime/pollForChanges.test.ts new file mode 100644 index 000000000..abfe9be68 --- /dev/null +++ b/library/agent/realtime/pollForChanges.test.ts @@ -0,0 +1,216 @@ +import * as t from "tap"; +import * as fetch from "../../helpers/fetch"; +import { wrap } from "../../helpers/wrap"; +import { Token } from "../api/Token"; +import { LoggerForTesting } from "../logger/LoggerForTesting"; +import { LoggerNoop } from "../logger/LoggerNoop"; +import { pollForChanges } from "./pollForChanges"; +import * as FakeTimers from "@sinonjs/fake-timers"; +import { Config } from "../Config"; + +t.test("it does not start interval if no token", async (t) => { + const logger = new LoggerForTesting(); + pollForChanges({ + onConfigUpdate: (config) => t.fail(), + logger: logger, + token: undefined, + lastUpdatedAt: 0, + }); + + t.same(logger.getMessages(), [ + "No token provided, not polling for config updates", + ]); +}); + +t.test("it checks for config updates", async () => { + const clock = FakeTimers.install(); + + const calls: { url: string; method: string }[] = []; + let configUpdatedAt = 0; + + wrap(fetch, "fetch", function fetch() { + return async function fetch(params: any) { + calls.push({ + url: params.url.toString(), + method: params.method, + }); + + if (params.url.hostname.startsWith("runtime")) { + return { + body: JSON.stringify({ + configUpdatedAt: configUpdatedAt, + }), + statusCode: 200, + }; + } + + if (params.url.hostname.startsWith("guard")) { + return { + body: JSON.stringify({ + endpoints: [], + heartbeatIntervalInMS: 10 * 60 * 1000, + configUpdatedAt: configUpdatedAt, + }), + statusCode: 200, + }; + } + + throw new Error(`Unknown hostname: ${params.url.hostname}`); + }; + }); + + const configUpdates: Config[] = []; + + pollForChanges({ + onConfigUpdate: (config) => { + configUpdates.push(config); + }, + logger: new LoggerNoop(), + token: new Token("123"), + lastUpdatedAt: 0, + }); + + t.same(configUpdates, []); + t.same(calls, []); + + await clock.nextAsync(); + + t.same(configUpdates, []); + t.same(calls, [ + { + url: "https://runtime.aikido.dev/config", + method: "GET", + }, + ]); + + configUpdatedAt = 1; + await clock.nextAsync(); + + t.same(configUpdates, [ + { + endpoints: [], + heartbeatIntervalInMS: 10 * 60 * 1000, + configUpdatedAt: 1, + }, + ]); + t.same(calls, [ + { + url: "https://runtime.aikido.dev/config", + method: "GET", + }, + { + url: "https://runtime.aikido.dev/config", + method: "GET", + }, + { + url: "https://guard.aikido.dev/api/runtime/config", + method: "GET", + }, + ]); + + await clock.nextAsync(); + + t.same(configUpdates, [ + { + endpoints: [], + heartbeatIntervalInMS: 10 * 60 * 1000, + configUpdatedAt: 1, + }, + ]); + t.same(calls, [ + { + url: "https://runtime.aikido.dev/config", + method: "GET", + }, + { + url: "https://runtime.aikido.dev/config", + method: "GET", + }, + { + url: "https://guard.aikido.dev/api/runtime/config", + method: "GET", + }, + { + url: "https://runtime.aikido.dev/config", + method: "GET", + }, + ]); + + configUpdatedAt = 2; + await clock.nextAsync(); + + t.same(configUpdates, [ + { + endpoints: [], + heartbeatIntervalInMS: 10 * 60 * 1000, + configUpdatedAt: 1, + }, + { + endpoints: [], + heartbeatIntervalInMS: 10 * 60 * 1000, + configUpdatedAt: 2, + }, + ]); + t.same(calls, [ + { + url: "https://runtime.aikido.dev/config", + method: "GET", + }, + { + url: "https://runtime.aikido.dev/config", + method: "GET", + }, + { + url: "https://guard.aikido.dev/api/runtime/config", + method: "GET", + }, + { + url: "https://runtime.aikido.dev/config", + method: "GET", + }, + { + url: "https://runtime.aikido.dev/config", + method: "GET", + }, + { + url: "https://guard.aikido.dev/api/runtime/config", + method: "GET", + }, + ]); + + clock.uninstall(); +}); + +t.test("it deals with API throwing errors", async () => { + const clock = FakeTimers.install(); + + wrap(fetch, "fetch", function fetch() { + return async function fetch() { + throw new Error("Request timed out"); + }; + }); + + const configUpdates: Config[] = []; + + const logger = new LoggerForTesting(); + pollForChanges({ + onConfigUpdate: (config) => { + configUpdates.push(config); + }, + logger: logger, + token: new Token("123"), + lastUpdatedAt: 0, + }); + + t.same(configUpdates, []); + t.same(logger.getMessages(), []); + + await clock.nextAsync(); + + t.same(configUpdates, []); + t.same(logger.getMessages(), [ + `Failed to check for config updates: Request timed out`, + ]); + + clock.uninstall(); +}); diff --git a/library/agent/realtime/pollForChanges.ts b/library/agent/realtime/pollForChanges.ts new file mode 100644 index 000000000..dd025e3b7 --- /dev/null +++ b/library/agent/realtime/pollForChanges.ts @@ -0,0 +1,49 @@ +import type { Token } from "../api/Token"; +import type { ConfigUpdateOptions } from "./ConfigUpdateOptions"; +import { getConfig } from "./getConfig"; +import { getConfigLastUpdatedAt } from "./getConfigLastUpdatedAt"; + +let interval: NodeJS.Timeout | null = null; +let currentLastUpdatedAt: number | null = null; + +export function pollForChanges({ + onConfigUpdate, + token, + logger, + lastUpdatedAt, +}: ConfigUpdateOptions) { + if (!token) { + logger.log("No token provided, not polling for config updates"); + return; + } + + currentLastUpdatedAt = lastUpdatedAt; + + if (interval) { + clearInterval(interval); + } + + interval = setInterval(() => { + check(token, onConfigUpdate).catch((error) => { + logger.log(`Failed to check for config updates: ${error.message}`); + }); + }, 60 * 1000); + + interval.unref(); +} + +async function check( + token: Token, + onConfigUpdate: ConfigUpdateOptions["onConfigUpdate"] +) { + const configLastUpdatedAt = await getConfigLastUpdatedAt(token); + + if ( + typeof currentLastUpdatedAt === "number" && + configLastUpdatedAt > currentLastUpdatedAt + ) { + const config = await getConfig(token); + currentLastUpdatedAt = config.configUpdatedAt; + onConfigUpdate(config); + } +} From 713b86da50abd890b8b20fc7160dea82fefdc33f Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Fri, 15 May 2026 18:19:21 +0200 Subject: [PATCH 11/47] Use zen.aikido.dev as default realtime hostname runtime.aikido.dev is replaced by zen.aikido.dev for config polling and SSE connections. On startup the agent probes zen.aikido.dev/config and falls back to runtime.aikido.dev (polling only, no SSE) when the host is unreachable, so environments with outbound allowlists keep working. --- library/agent/Agent.ts | 20 +++++++++-- library/agent/realtime/ConfigUpdateOptions.ts | 1 + library/agent/realtime/connectToSSE.ts | 5 +-- .../agent/realtime/getConfigLastUpdatedAt.ts | 8 +++-- library/agent/realtime/getRealtimeURL.ts | 2 +- .../agent/realtime/listenForConfigUpdates.ts | 2 ++ library/agent/realtime/pollForChanges.test.ts | 25 +++++++------ library/agent/realtime/pollForChanges.ts | 6 ++-- library/agent/realtime/resolveRealtimeURL.ts | 36 +++++++++++++++++++ library/sinks/HTTPRequest.got.test.ts | 5 +-- 10 files changed, 84 insertions(+), 26 deletions(-) create mode 100644 library/agent/realtime/resolveRealtimeURL.ts diff --git a/library/agent/Agent.ts b/library/agent/Agent.ts index 4d6eea907..08f71b77a 100644 --- a/library/agent/Agent.ts +++ b/library/agent/Agent.ts @@ -40,6 +40,8 @@ import { PendingEvents } from "./PendingEvents"; import type { IdorProtectionConfig } from "./IdorProtectionConfig"; import { warnIfTsxIsUsed } from "../helpers/warnIfTsxIsUsed"; import { pollForChanges } from "./realtime/pollForChanges"; +import { getRealtimeURL } from "./realtime/getRealtimeURL"; +import { resolveRealtimeURL } from "./realtime/resolveRealtimeURL"; type WrappedPackage = { version: string; supported: boolean }; @@ -454,11 +456,18 @@ export class Agent { } } - private startCheckingForConfigUpdates() { + private async startCheckingForConfigUpdates() { + if (!this.token) { + return; + } + + const realtimeURL = await resolveRealtimeURL(this.token, this.logger); + const options: ConfigUpdateOptions = { token: this.token, logger: this.logger, lastUpdatedAt: this.serviceConfig.getLastUpdatedAt(), + realtimeURL, onConfigUpdate: (config) => { this.updateServiceConfig({ success: true, ...config }); this.updateBlockedLists().catch((error) => { @@ -467,7 +476,10 @@ export class Agent { }, }; - listenForConfigUpdates(options); + if (realtimeURL.hostname !== "runtime.aikido.dev") { + listenForConfigUpdates(options); + } + pollForChanges(options); } @@ -772,7 +784,9 @@ export class Agent { } const promise = sendUserEvent(this.token, event).catch(() => { - this.logger.log("Failed to report tracked event"); + this.logger.log( + `Failed to send tracked event, ensure ${getRealtimeURL().hostname} is in your outbound firewall allowlist` + ); }); this.pendingEvents.onAPICall(promise); } diff --git a/library/agent/realtime/ConfigUpdateOptions.ts b/library/agent/realtime/ConfigUpdateOptions.ts index 2129c9833..60d9209c8 100644 --- a/library/agent/realtime/ConfigUpdateOptions.ts +++ b/library/agent/realtime/ConfigUpdateOptions.ts @@ -7,4 +7,5 @@ export type ConfigUpdateOptions = { token: Token | undefined; logger: Logger; lastUpdatedAt: number; + realtimeURL: URL; }; diff --git a/library/agent/realtime/connectToSSE.ts b/library/agent/realtime/connectToSSE.ts index 26396a8f6..3d64bc0da 100644 --- a/library/agent/realtime/connectToSSE.ts +++ b/library/agent/realtime/connectToSSE.ts @@ -5,7 +5,6 @@ import type { EventSourceMessage } from "../../helpers/eventsource-parser/types" import { isDebuggingSSE } from "../../helpers/isDebuggingSSE"; import { Token } from "../api/Token"; import { Logger } from "../logger/Logger"; -import { getRealtimeURL } from "./getRealtimeURL"; const INITIAL_RECONNECT_MS = 1000; const MAX_RECONNECT_MS = 60 * 1000; @@ -13,10 +12,12 @@ const MAX_RECONNECT_MS = 60 * 1000; export function connectToSSE({ token, logger, + realtimeURL, onEvent, }: { token: Token; logger: Logger; + realtimeURL: URL; onEvent: (event: EventSourceMessage) => void; }) { let reconnectMs = INITIAL_RECONNECT_MS; @@ -31,7 +32,7 @@ export function connectToSSE({ currentRequest = null; } - const url = new URL(`${getRealtimeURL().toString()}api/runtime/stream`); + const url = new URL(`${realtimeURL.toString()}api/runtime/stream`); if (debugSSE) { logger.log(`SSE connecting to ${url.toString()}`); diff --git a/library/agent/realtime/getConfigLastUpdatedAt.ts b/library/agent/realtime/getConfigLastUpdatedAt.ts index 28f6512fa..741acc434 100644 --- a/library/agent/realtime/getConfigLastUpdatedAt.ts +++ b/library/agent/realtime/getConfigLastUpdatedAt.ts @@ -1,12 +1,14 @@ import { fetch } from "../../helpers/fetch"; import { Token } from "../api/Token"; -import { getRealtimeURL } from "./getRealtimeURL"; type RealtimeResponse = { configUpdatedAt: number }; -export async function getConfigLastUpdatedAt(token: Token): Promise { +export async function getConfigLastUpdatedAt( + token: Token, + realtimeURL: URL +): Promise { const { body, statusCode } = await fetch({ - url: new URL(`${getRealtimeURL().toString()}config`), + url: new URL(`${realtimeURL.toString()}config`), method: "GET", headers: { Authorization: token.asString(), diff --git a/library/agent/realtime/getRealtimeURL.ts b/library/agent/realtime/getRealtimeURL.ts index 6f2cc465d..a177ddf34 100644 --- a/library/agent/realtime/getRealtimeURL.ts +++ b/library/agent/realtime/getRealtimeURL.ts @@ -3,5 +3,5 @@ export function getRealtimeURL() { return new URL(process.env.AIKIDO_REALTIME_ENDPOINT); } - return new URL("https://runtime.aikido.dev"); + return new URL("https://zen.aikido.dev"); } diff --git a/library/agent/realtime/listenForConfigUpdates.ts b/library/agent/realtime/listenForConfigUpdates.ts index d493d3e99..96c5fac50 100644 --- a/library/agent/realtime/listenForConfigUpdates.ts +++ b/library/agent/realtime/listenForConfigUpdates.ts @@ -8,6 +8,7 @@ export function listenForConfigUpdates({ token, logger, lastUpdatedAt, + realtimeURL, }: ConfigUpdateOptions) { if (!token) { logger.log("No token provided, not listening for config updates"); @@ -21,6 +22,7 @@ export function listenForConfigUpdates({ connectToSSE({ token, logger, + realtimeURL, onEvent(event) { if (debugSSE) { logger.log(`SSE event received: ${event.event}`); diff --git a/library/agent/realtime/pollForChanges.test.ts b/library/agent/realtime/pollForChanges.test.ts index abfe9be68..a77cb81f2 100644 --- a/library/agent/realtime/pollForChanges.test.ts +++ b/library/agent/realtime/pollForChanges.test.ts @@ -15,6 +15,7 @@ t.test("it does not start interval if no token", async (t) => { logger: logger, token: undefined, lastUpdatedAt: 0, + realtimeURL: new URL("https://zen.aikido.dev"), }); t.same(logger.getMessages(), [ @@ -35,7 +36,7 @@ t.test("it checks for config updates", async () => { method: params.method, }); - if (params.url.hostname.startsWith("runtime")) { + if (params.url.hostname.startsWith("zen")) { return { body: JSON.stringify({ configUpdatedAt: configUpdatedAt, @@ -68,6 +69,7 @@ t.test("it checks for config updates", async () => { logger: new LoggerNoop(), token: new Token("123"), lastUpdatedAt: 0, + realtimeURL: new URL("https://zen.aikido.dev"), }); t.same(configUpdates, []); @@ -78,7 +80,7 @@ t.test("it checks for config updates", async () => { t.same(configUpdates, []); t.same(calls, [ { - url: "https://runtime.aikido.dev/config", + url: "https://zen.aikido.dev/config", method: "GET", }, ]); @@ -95,11 +97,11 @@ t.test("it checks for config updates", async () => { ]); t.same(calls, [ { - url: "https://runtime.aikido.dev/config", + url: "https://zen.aikido.dev/config", method: "GET", }, { - url: "https://runtime.aikido.dev/config", + url: "https://zen.aikido.dev/config", method: "GET", }, { @@ -119,11 +121,11 @@ t.test("it checks for config updates", async () => { ]); t.same(calls, [ { - url: "https://runtime.aikido.dev/config", + url: "https://zen.aikido.dev/config", method: "GET", }, { - url: "https://runtime.aikido.dev/config", + url: "https://zen.aikido.dev/config", method: "GET", }, { @@ -131,7 +133,7 @@ t.test("it checks for config updates", async () => { method: "GET", }, { - url: "https://runtime.aikido.dev/config", + url: "https://zen.aikido.dev/config", method: "GET", }, ]); @@ -153,11 +155,11 @@ t.test("it checks for config updates", async () => { ]); t.same(calls, [ { - url: "https://runtime.aikido.dev/config", + url: "https://zen.aikido.dev/config", method: "GET", }, { - url: "https://runtime.aikido.dev/config", + url: "https://zen.aikido.dev/config", method: "GET", }, { @@ -165,11 +167,11 @@ t.test("it checks for config updates", async () => { method: "GET", }, { - url: "https://runtime.aikido.dev/config", + url: "https://zen.aikido.dev/config", method: "GET", }, { - url: "https://runtime.aikido.dev/config", + url: "https://zen.aikido.dev/config", method: "GET", }, { @@ -200,6 +202,7 @@ t.test("it deals with API throwing errors", async () => { logger: logger, token: new Token("123"), lastUpdatedAt: 0, + realtimeURL: new URL("https://zen.aikido.dev"), }); t.same(configUpdates, []); diff --git a/library/agent/realtime/pollForChanges.ts b/library/agent/realtime/pollForChanges.ts index dd025e3b7..0af1d02f5 100644 --- a/library/agent/realtime/pollForChanges.ts +++ b/library/agent/realtime/pollForChanges.ts @@ -11,6 +11,7 @@ export function pollForChanges({ token, logger, lastUpdatedAt, + realtimeURL, }: ConfigUpdateOptions) { if (!token) { logger.log("No token provided, not polling for config updates"); @@ -24,7 +25,7 @@ export function pollForChanges({ } interval = setInterval(() => { - check(token, onConfigUpdate).catch((error) => { + check(token, realtimeURL, onConfigUpdate).catch((error) => { logger.log(`Failed to check for config updates: ${error.message}`); }); }, 60 * 1000); @@ -34,9 +35,10 @@ export function pollForChanges({ async function check( token: Token, + realtimeURL: URL, onConfigUpdate: ConfigUpdateOptions["onConfigUpdate"] ) { - const configLastUpdatedAt = await getConfigLastUpdatedAt(token); + const configLastUpdatedAt = await getConfigLastUpdatedAt(token, realtimeURL); if ( typeof currentLastUpdatedAt === "number" && diff --git a/library/agent/realtime/resolveRealtimeURL.ts b/library/agent/realtime/resolveRealtimeURL.ts new file mode 100644 index 000000000..9284090ab --- /dev/null +++ b/library/agent/realtime/resolveRealtimeURL.ts @@ -0,0 +1,36 @@ +import { fetch } from "../../helpers/fetch"; +import type { Token } from "../api/Token"; +import type { Logger } from "../logger/Logger"; +import { getRealtimeURL } from "./getRealtimeURL"; + +const FALLBACK_URL = "https://runtime.aikido.dev"; + +export async function resolveRealtimeURL( + token: Token, + logger: Logger +): Promise { + const realtimeURL = getRealtimeURL(); + + if (process.env.AIKIDO_REALTIME_ENDPOINT) { + return realtimeURL; + } + + try { + await fetch({ + url: new URL(`${realtimeURL.toString()}config`), + method: "GET", + headers: { + Authorization: token.asString(), + }, + timeoutInMS: 5000, + }); + + return realtimeURL; + } catch { + logger.log( + `Unable to reach ${realtimeURL.hostname}, falling back to ${FALLBACK_URL}. Realtime updates (SSE) will not be available, using polling instead.` + ); + + return new URL(FALLBACK_URL); + } +} diff --git a/library/sinks/HTTPRequest.got.test.ts b/library/sinks/HTTPRequest.got.test.ts index 818103916..0435c219c 100644 --- a/library/sinks/HTTPRequest.got.test.ts +++ b/library/sinks/HTTPRequest.got.test.ts @@ -1,5 +1,4 @@ import * as t from "tap"; -import { Token } from "../agent/api/Token"; import { Context, runWithContext } from "../agent/Context"; import { HTTPRequest } from "./HTTPRequest"; import { createTestAgent } from "../helpers/createTestAgent"; @@ -51,9 +50,7 @@ t.before(async () => { }); t.test("it works", opts, async (t) => { - const agent = createTestAgent({ - token: new Token("123"), - }); + const agent = createTestAgent(); agent.start([new HTTPRequest()]); t.same(agent.getHostnames().asArray(), []); From e58b480abe95df432ee9b6379e39c91662cba746 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Tue, 19 May 2026 17:26:39 +0200 Subject: [PATCH 12/47] Use getRealtimeURL directly for SSE connections SSE always connects to zen.aikido.dev via getRealtimeURL() and handles failures through its own exponential backoff. The polling URL resolve (with runtime.aikido.dev fallback) is separated into resolvePollingURL so the two concerns are independent. --- library/agent/Agent.ts | 9 +++------ library/agent/realtime/connectToSSE.ts | 5 ++--- library/agent/realtime/listenForConfigUpdates.ts | 2 -- .../{resolveRealtimeURL.ts => resolvePollingURL.ts} | 2 +- 4 files changed, 6 insertions(+), 12 deletions(-) rename library/agent/realtime/{resolveRealtimeURL.ts => resolvePollingURL.ts} (95%) diff --git a/library/agent/Agent.ts b/library/agent/Agent.ts index 08f71b77a..bf081da25 100644 --- a/library/agent/Agent.ts +++ b/library/agent/Agent.ts @@ -41,7 +41,7 @@ import type { IdorProtectionConfig } from "./IdorProtectionConfig"; import { warnIfTsxIsUsed } from "../helpers/warnIfTsxIsUsed"; import { pollForChanges } from "./realtime/pollForChanges"; import { getRealtimeURL } from "./realtime/getRealtimeURL"; -import { resolveRealtimeURL } from "./realtime/resolveRealtimeURL"; +import { resolvePollingURL } from "./realtime/resolvePollingURL"; type WrappedPackage = { version: string; supported: boolean }; @@ -461,7 +461,7 @@ export class Agent { return; } - const realtimeURL = await resolveRealtimeURL(this.token, this.logger); + const realtimeURL = await resolvePollingURL(this.token, this.logger); const options: ConfigUpdateOptions = { token: this.token, @@ -476,10 +476,7 @@ export class Agent { }, }; - if (realtimeURL.hostname !== "runtime.aikido.dev") { - listenForConfigUpdates(options); - } - + listenForConfigUpdates(options); pollForChanges(options); } diff --git a/library/agent/realtime/connectToSSE.ts b/library/agent/realtime/connectToSSE.ts index 3d64bc0da..26396a8f6 100644 --- a/library/agent/realtime/connectToSSE.ts +++ b/library/agent/realtime/connectToSSE.ts @@ -5,6 +5,7 @@ import type { EventSourceMessage } from "../../helpers/eventsource-parser/types" import { isDebuggingSSE } from "../../helpers/isDebuggingSSE"; import { Token } from "../api/Token"; import { Logger } from "../logger/Logger"; +import { getRealtimeURL } from "./getRealtimeURL"; const INITIAL_RECONNECT_MS = 1000; const MAX_RECONNECT_MS = 60 * 1000; @@ -12,12 +13,10 @@ const MAX_RECONNECT_MS = 60 * 1000; export function connectToSSE({ token, logger, - realtimeURL, onEvent, }: { token: Token; logger: Logger; - realtimeURL: URL; onEvent: (event: EventSourceMessage) => void; }) { let reconnectMs = INITIAL_RECONNECT_MS; @@ -32,7 +31,7 @@ export function connectToSSE({ currentRequest = null; } - const url = new URL(`${realtimeURL.toString()}api/runtime/stream`); + const url = new URL(`${getRealtimeURL().toString()}api/runtime/stream`); if (debugSSE) { logger.log(`SSE connecting to ${url.toString()}`); diff --git a/library/agent/realtime/listenForConfigUpdates.ts b/library/agent/realtime/listenForConfigUpdates.ts index 96c5fac50..d493d3e99 100644 --- a/library/agent/realtime/listenForConfigUpdates.ts +++ b/library/agent/realtime/listenForConfigUpdates.ts @@ -8,7 +8,6 @@ export function listenForConfigUpdates({ token, logger, lastUpdatedAt, - realtimeURL, }: ConfigUpdateOptions) { if (!token) { logger.log("No token provided, not listening for config updates"); @@ -22,7 +21,6 @@ export function listenForConfigUpdates({ connectToSSE({ token, logger, - realtimeURL, onEvent(event) { if (debugSSE) { logger.log(`SSE event received: ${event.event}`); diff --git a/library/agent/realtime/resolveRealtimeURL.ts b/library/agent/realtime/resolvePollingURL.ts similarity index 95% rename from library/agent/realtime/resolveRealtimeURL.ts rename to library/agent/realtime/resolvePollingURL.ts index 9284090ab..57d1cbdaa 100644 --- a/library/agent/realtime/resolveRealtimeURL.ts +++ b/library/agent/realtime/resolvePollingURL.ts @@ -5,7 +5,7 @@ import { getRealtimeURL } from "./getRealtimeURL"; const FALLBACK_URL = "https://runtime.aikido.dev"; -export async function resolveRealtimeURL( +export async function resolvePollingURL( token: Token, logger: Logger ): Promise { From f279f5be6e3d3ff13d1b0668a23c46ca519b07a9 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Tue, 19 May 2026 17:38:01 +0200 Subject: [PATCH 13/47] Add read timeout to SSE client --- library/agent/realtime/connectToSSE.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/library/agent/realtime/connectToSSE.ts b/library/agent/realtime/connectToSSE.ts index 26396a8f6..5bc66f56f 100644 --- a/library/agent/realtime/connectToSSE.ts +++ b/library/agent/realtime/connectToSSE.ts @@ -9,6 +9,7 @@ import { getRealtimeURL } from "./getRealtimeURL"; const INITIAL_RECONNECT_MS = 1000; const MAX_RECONNECT_MS = 60 * 1000; +const READ_TIMEOUT_MS = 70 * 1000; export function connectToSSE({ token, @@ -96,6 +97,13 @@ export function connectToSSE({ currentRequest = req; req.on("socket", (socket) => { + socket.setTimeout(READ_TIMEOUT_MS, () => { + if (debugSSE) { + logger.log("SSE read timeout, reconnecting"); + } + req.destroy(); + scheduleReconnect(); + }); socket.unref(); }); From 7c62ffd1a8067f7f2fb5def789dd5814c208bef1 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Tue, 19 May 2026 17:46:54 +0200 Subject: [PATCH 14/47] Inline ConfigUpdateOptions into each consumer SSE and polling need different args, so a shared type was hiding that. Each function now declares its own params and Agent.ts calls them separately with only the fields they need. --- library/agent/Agent.ts | 37 +++++++++++-------- library/agent/realtime/ConfigUpdateOptions.ts | 11 ------ .../agent/realtime/listenForConfigUpdates.ts | 13 ++++++- library/agent/realtime/pollForChanges.ts | 15 ++++++-- 4 files changed, 45 insertions(+), 31 deletions(-) delete mode 100644 library/agent/realtime/ConfigUpdateOptions.ts diff --git a/library/agent/Agent.ts b/library/agent/Agent.ts index bf081da25..7798a4c66 100644 --- a/library/agent/Agent.ts +++ b/library/agent/Agent.ts @@ -16,9 +16,8 @@ import type { import { sendUserEvent, type UserEvent } from "./api/UserEventsAPI"; import { Token } from "./api/Token"; import { Kind } from "./Attack"; -import { Endpoint } from "./Config"; +import { type Config, Endpoint } from "./Config"; import { listenForConfigUpdates } from "./realtime/listenForConfigUpdates"; -import type { ConfigUpdateOptions } from "./realtime/ConfigUpdateOptions"; import { Context } from "./Context"; import { Hostnames } from "./Hostnames"; import { InspectionStatistics } from "./InspectionStatistics"; @@ -461,23 +460,31 @@ export class Agent { return; } - const realtimeURL = await resolvePollingURL(this.token, this.logger); + const onConfigUpdate = (config: Config) => { + this.updateServiceConfig({ success: true, ...config }); + this.updateBlockedLists().catch((error) => { + this.logger.log(`Failed to update blocked lists: ${error.message}`); + }); + }; + + const lastUpdatedAt = this.serviceConfig.getLastUpdatedAt(); - const options: ConfigUpdateOptions = { + listenForConfigUpdates({ token: this.token, logger: this.logger, - lastUpdatedAt: this.serviceConfig.getLastUpdatedAt(), - realtimeURL, - onConfigUpdate: (config) => { - this.updateServiceConfig({ success: true, ...config }); - this.updateBlockedLists().catch((error) => { - this.logger.log(`Failed to update blocked lists: ${error.message}`); - }); - }, - }; + lastUpdatedAt, + onConfigUpdate, + }); + + const pollingURL = await resolvePollingURL(this.token, this.logger); - listenForConfigUpdates(options); - pollForChanges(options); + pollForChanges({ + token: this.token, + logger: this.logger, + lastUpdatedAt, + realtimeURL: pollingURL, + onConfigUpdate, + }); } private getHostname() { diff --git a/library/agent/realtime/ConfigUpdateOptions.ts b/library/agent/realtime/ConfigUpdateOptions.ts deleted file mode 100644 index 60d9209c8..000000000 --- a/library/agent/realtime/ConfigUpdateOptions.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { Token } from "../api/Token"; -import type { Config } from "../Config"; -import type { Logger } from "../logger/Logger"; - -export type ConfigUpdateOptions = { - onConfigUpdate: (config: Config) => void; - token: Token | undefined; - logger: Logger; - lastUpdatedAt: number; - realtimeURL: URL; -}; diff --git a/library/agent/realtime/listenForConfigUpdates.ts b/library/agent/realtime/listenForConfigUpdates.ts index d493d3e99..a2226be90 100644 --- a/library/agent/realtime/listenForConfigUpdates.ts +++ b/library/agent/realtime/listenForConfigUpdates.ts @@ -1,14 +1,23 @@ +import type { Config } from "../Config"; +import type { Token } from "../api/Token"; +import type { Logger } from "../logger/Logger"; import { isDebuggingSSE } from "../../helpers/isDebuggingSSE"; import { connectToSSE } from "./connectToSSE"; -import type { ConfigUpdateOptions } from "./ConfigUpdateOptions"; import { getConfig } from "./getConfig"; +type OnConfigUpdate = (config: Config) => void; + export function listenForConfigUpdates({ onConfigUpdate, token, logger, lastUpdatedAt, -}: ConfigUpdateOptions) { +}: { + onConfigUpdate: OnConfigUpdate; + token: Token | undefined; + logger: Logger; + lastUpdatedAt: number; +}) { if (!token) { logger.log("No token provided, not listening for config updates"); return; diff --git a/library/agent/realtime/pollForChanges.ts b/library/agent/realtime/pollForChanges.ts index 0af1d02f5..7add02c02 100644 --- a/library/agent/realtime/pollForChanges.ts +++ b/library/agent/realtime/pollForChanges.ts @@ -1,8 +1,11 @@ +import type { Config } from "../Config"; import type { Token } from "../api/Token"; -import type { ConfigUpdateOptions } from "./ConfigUpdateOptions"; +import type { Logger } from "../logger/Logger"; import { getConfig } from "./getConfig"; import { getConfigLastUpdatedAt } from "./getConfigLastUpdatedAt"; +type OnConfigUpdate = (config: Config) => void; + let interval: NodeJS.Timeout | null = null; let currentLastUpdatedAt: number | null = null; @@ -12,7 +15,13 @@ export function pollForChanges({ logger, lastUpdatedAt, realtimeURL, -}: ConfigUpdateOptions) { +}: { + onConfigUpdate: OnConfigUpdate; + token: Token | undefined; + logger: Logger; + lastUpdatedAt: number; + realtimeURL: URL; +}) { if (!token) { logger.log("No token provided, not polling for config updates"); return; @@ -36,7 +45,7 @@ export function pollForChanges({ async function check( token: Token, realtimeURL: URL, - onConfigUpdate: ConfigUpdateOptions["onConfigUpdate"] + onConfigUpdate: OnConfigUpdate ) { const configLastUpdatedAt = await getConfigLastUpdatedAt(token, realtimeURL); From c36921ce56e0a2c869e34cffc32ee6d63cfbfb27 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Tue, 19 May 2026 17:47:48 +0200 Subject: [PATCH 15/47] Inline pollingURL variable --- library/agent/Agent.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/library/agent/Agent.ts b/library/agent/Agent.ts index 7798a4c66..5d9e9acce 100644 --- a/library/agent/Agent.ts +++ b/library/agent/Agent.ts @@ -476,13 +476,11 @@ export class Agent { onConfigUpdate, }); - const pollingURL = await resolvePollingURL(this.token, this.logger); - pollForChanges({ token: this.token, logger: this.logger, lastUpdatedAt, - realtimeURL: pollingURL, + realtimeURL: await resolvePollingURL(this.token, this.logger), onConfigUpdate, }); } From d0e97c6ce5138cf647be87649165e93554ca4e9c Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Tue, 19 May 2026 18:04:13 +0200 Subject: [PATCH 16/47] Retry polling URL probe with backoff A momentary blip during startup would permanently fall back to runtime.aikido.dev for the entire process lifetime. Retry up to 3 times with exponential backoff before giving up. --- library/agent/realtime/resolvePollingURL.ts | 47 ++++++++++++++------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/library/agent/realtime/resolvePollingURL.ts b/library/agent/realtime/resolvePollingURL.ts index 57d1cbdaa..03e68da1d 100644 --- a/library/agent/realtime/resolvePollingURL.ts +++ b/library/agent/realtime/resolvePollingURL.ts @@ -4,6 +4,24 @@ import type { Logger } from "../logger/Logger"; import { getRealtimeURL } from "./getRealtimeURL"; const FALLBACK_URL = "https://runtime.aikido.dev"; +const MAX_RETRIES = 3; + +async function probe(url: URL, token: Token): Promise { + try { + await fetch({ + url, + method: "GET", + headers: { + Authorization: token.asString(), + }, + timeoutInMS: 5000, + }); + + return true; + } catch { + return false; + } +} export async function resolvePollingURL( token: Token, @@ -15,22 +33,21 @@ export async function resolvePollingURL( return realtimeURL; } - try { - await fetch({ - url: new URL(`${realtimeURL.toString()}config`), - method: "GET", - headers: { - Authorization: token.asString(), - }, - timeoutInMS: 5000, - }); + const configURL = new URL(`${realtimeURL.toString()}config`); + let backoffMs = 1000; - return realtimeURL; - } catch { - logger.log( - `Unable to reach ${realtimeURL.hostname}, falling back to ${FALLBACK_URL}. Realtime updates (SSE) will not be available, using polling instead.` - ); + for (let i = 0; i < MAX_RETRIES; i++) { + if (await probe(configURL, token)) { + return realtimeURL; + } - return new URL(FALLBACK_URL); + await new Promise((resolve) => setTimeout(resolve, backoffMs)); + backoffMs *= 2; } + + logger.log( + `Unable to reach ${realtimeURL.hostname}, falling back to ${FALLBACK_URL}. Realtime updates (SSE) will not be available, using polling instead.` + ); + + return new URL(FALLBACK_URL); } From 4d6aeb065b400bdcf0964c93a679ecabcc44a79f Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Tue, 19 May 2026 18:08:33 +0200 Subject: [PATCH 17/47] Simplify SSE reconnect backoff Remove jitter and inline the delay variable. --- library/agent/realtime/connectToSSE.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/library/agent/realtime/connectToSSE.ts b/library/agent/realtime/connectToSSE.ts index 5bc66f56f..1ac3f1c3a 100644 --- a/library/agent/realtime/connectToSSE.ts +++ b/library/agent/realtime/connectToSSE.ts @@ -120,16 +120,12 @@ export function connectToSSE({ clearTimeout(reconnectTimer); } - // Exponential backoff with jitter - const jitter = Math.random() * 0.5 + 0.75; // 0.75 - 1.25 - const delay = Math.min(reconnectMs * jitter, MAX_RECONNECT_MS); - reconnectMs = Math.min(reconnectMs * 2, MAX_RECONNECT_MS); - if (debugSSE) { - logger.log(`SSE scheduling reconnect in ${Math.round(delay)}ms`); + logger.log(`SSE scheduling reconnect in ${reconnectMs}ms`); } - reconnectTimer = setTimeout(connect, delay); + reconnectTimer = setTimeout(connect, reconnectMs); + reconnectMs = Math.min(reconnectMs * 2, MAX_RECONNECT_MS); reconnectTimer.unref(); } From 692609e76b25efda6001a4f74f289f2362ffc97b Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Wed, 20 May 2026 09:58:34 +0200 Subject: [PATCH 18/47] Add SSE stream endpoint to mock server and e2e test --- end2end/server/app.ts | 2 + end2end/server/src/handlers/stream.ts | 36 +++++++ end2end/server/src/zen/config.ts | 4 + .../realtime-config-updates.test.mjs | 96 +++++++++++++++++++ library/agent/Agent.ts | 2 +- library/agent/realtime/pollForChanges.ts | 6 +- library/agent/realtime/resolvePollingURL.ts | 5 +- 7 files changed, 144 insertions(+), 7 deletions(-) create mode 100644 end2end/server/src/handlers/stream.ts create mode 100644 end2end/tests-new/realtime-config-updates.test.mjs diff --git a/end2end/server/app.ts b/end2end/server/app.ts index da888d0a0..4aa6cf638 100644 --- a/end2end/server/app.ts +++ b/end2end/server/app.ts @@ -10,6 +10,7 @@ import { updateConfig } from "./src/handlers/updateConfig.ts"; import { lists } from "./src/handlers/lists.ts"; import { updateIPLists } from "./src/handlers/updateLists.ts"; import { realtimeConfig } from "./src/handlers/realtimeConfig.ts"; +import { stream } from "./src/handlers/stream.ts"; const app = express(); app.set("trust proxy", false); @@ -24,6 +25,7 @@ app.post("/api/runtime/config", checkToken, updateConfig); // Realtime polling endpoint app.get("/config", checkToken, realtimeConfig); +app.get("/api/runtime/stream", checkToken, stream); app.get("/api/runtime/events", checkToken, listEvents); app.post("/api/runtime/events", checkToken, captureEvent); diff --git a/end2end/server/src/handlers/stream.ts b/end2end/server/src/handlers/stream.ts new file mode 100644 index 000000000..a701a4928 --- /dev/null +++ b/end2end/server/src/handlers/stream.ts @@ -0,0 +1,36 @@ +import type { Response } from "express"; +import { getAppConfig, configEvents } from "../zen/config.ts"; +import type { ZenRequest } from "../types.ts"; + +export function stream(req: ZenRequest, res: Response) { + if (!req.zenApp) { + throw new Error("App is missing"); + } + + const app = req.zenApp; + + res.writeHead(200, { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }); + + function sendConfig() { + const config = getAppConfig(app); + res.write(`event: config-updated\ndata: ${JSON.stringify(config)}\n\n`); + } + + sendConfig(); + + const eventName = `config-updated:${app.id}`; + configEvents.on(eventName, sendConfig); + + const ping = setInterval(() => { + res.write(": ping\n\n"); + }, 30_000); + + req.on("close", () => { + configEvents.off(eventName, sendConfig); + clearInterval(ping); + }); +} diff --git a/end2end/server/src/zen/config.ts b/end2end/server/src/zen/config.ts index c59198c20..cc81d6af4 100644 --- a/end2end/server/src/zen/config.ts +++ b/end2end/server/src/zen/config.ts @@ -1,5 +1,8 @@ +import { EventEmitter } from "node:events"; import type { App } from "./apps.ts"; +export const configEvents = new EventEmitter(); + type AppConfig = { success: boolean; serviceId: number; @@ -53,6 +56,7 @@ export function updateAppConfig(app: App, newConfig: Partial) { ...newConfig, configUpdatedAt: Date.now(), }; + configEvents.emit(`config-updated:${app.id}`); return true; } diff --git a/end2end/tests-new/realtime-config-updates.test.mjs b/end2end/tests-new/realtime-config-updates.test.mjs new file mode 100644 index 000000000..a91649b40 --- /dev/null +++ b/end2end/tests-new/realtime-config-updates.test.mjs @@ -0,0 +1,96 @@ +import { spawn } from "child_process"; +import { resolve } from "path"; +import { test } from "node:test"; +import { equal, fail } from "node:assert"; +import { getRandomPort } from "./utils/get-port.mjs"; +import { timeout } from "./utils/timeout.mjs"; + +const pathToAppDir = resolve( + import.meta.dirname, + "../../sample-apps/hono-pg-ts-esm" +); + +const testServerUrl = "http://localhost:5874"; +const port = await getRandomPort(); + +test("it picks up blocked IP via SSE config update", async () => { + const response = await fetch(`${testServerUrl}/api/runtime/apps`, { + method: "POST", + }); + const body = await response.json(); + const token = body.token; + + const server = spawn( + `node`, + [ + "--require", + "@aikidosec/firewall/instrument", + "--experimental-strip-types", + "./app.ts", + port, + ], + { + cwd: pathToAppDir, + env: { + ...process.env, + AIKIDO_TOKEN: token, + AIKIDO_ENDPOINT: testServerUrl, + AIKIDO_REALTIME_ENDPOINT: testServerUrl, + AIKIDO_DEBUG: "true", + AIKIDO_BLOCK: "true", + }, + } + ); + + try { + server.on("error", (err) => { + fail(err); + }); + + let stdout = ""; + server.stdout.on("data", (data) => { + stdout += data.toString(); + }); + + let stderr = ""; + server.stderr.on("data", (data) => { + stderr += data.toString(); + }); + + // Wait for the server to start and SSE to connect + await timeout(3000); + + // Verify request from 5.6.7.8 is allowed before blocking + const before = await fetch(`http://127.0.0.1:${port}/`, { + headers: { "x-forwarded-for": "5.6.7.8" }, + signal: AbortSignal.timeout(5000), + }); + equal(before.status, 200); + + // Block IP 5.6.7.8 via the test server API + await fetch(`${testServerUrl}/api/runtime/firewall/lists`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: token, + }, + body: JSON.stringify({ + blockedIPAddresses: ["5.6.7.8"], + }), + }); + + // Wait for SSE config-updated event to propagate + await timeout(2000); + + // Verify request from 5.6.7.8 is now blocked + const after = await fetch(`http://127.0.0.1:${port}/`, { + headers: { "x-forwarded-for": "5.6.7.8" }, + signal: AbortSignal.timeout(5000), + }); + equal(after.status, 403); + } catch (err) { + fail(err); + } finally { + server.kill(); + } +}); diff --git a/library/agent/Agent.ts b/library/agent/Agent.ts index 5d9e9acce..32b23a31a 100644 --- a/library/agent/Agent.ts +++ b/library/agent/Agent.ts @@ -787,7 +787,7 @@ export class Agent { const promise = sendUserEvent(this.token, event).catch(() => { this.logger.log( - `Failed to send tracked event, ensure ${getRealtimeURL().hostname} is in your outbound firewall allowlist` + `Can't send tracked event, make sure ${getRealtimeURL().hostname} is in your outbound firewall allowlist` ); }); this.pendingEvents.onAPICall(promise); diff --git a/library/agent/realtime/pollForChanges.ts b/library/agent/realtime/pollForChanges.ts index 7add02c02..5630e43bd 100644 --- a/library/agent/realtime/pollForChanges.ts +++ b/library/agent/realtime/pollForChanges.ts @@ -1,6 +1,6 @@ -import type { Config } from "../Config"; -import type { Token } from "../api/Token"; -import type { Logger } from "../logger/Logger"; +import { Token } from "../api/Token"; +import { Config } from "../Config"; +import { Logger } from "../logger/Logger"; import { getConfig } from "./getConfig"; import { getConfigLastUpdatedAt } from "./getConfigLastUpdatedAt"; diff --git a/library/agent/realtime/resolvePollingURL.ts b/library/agent/realtime/resolvePollingURL.ts index 03e68da1d..cfaaa3a6a 100644 --- a/library/agent/realtime/resolvePollingURL.ts +++ b/library/agent/realtime/resolvePollingURL.ts @@ -3,7 +3,6 @@ import type { Token } from "../api/Token"; import type { Logger } from "../logger/Logger"; import { getRealtimeURL } from "./getRealtimeURL"; -const FALLBACK_URL = "https://runtime.aikido.dev"; const MAX_RETRIES = 3; async function probe(url: URL, token: Token): Promise { @@ -46,8 +45,8 @@ export async function resolvePollingURL( } logger.log( - `Unable to reach ${realtimeURL.hostname}, falling back to ${FALLBACK_URL}. Realtime updates (SSE) will not be available, using polling instead.` + `Can't reach ${realtimeURL.hostname}, make sure it's in your outbound firewall allowlist. Realtime config updates won't be available, switched to polling.` ); - return new URL(FALLBACK_URL); + return new URL("https://runtime.aikido.dev"); } From 081846ccc2e0a74e268cdfbf1ac9475dd7cde9a0 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Wed, 20 May 2026 10:03:26 +0200 Subject: [PATCH 19/47] Add logDebug helper to SSE modules Gate all SSE log calls behind a single logDebug function that checks the AIKIDO_DEBUG_SSE flag, replacing repeated if-guards. --- library/agent/realtime/connectToSSE.ts | 38 +++++++++---------- .../agent/realtime/listenForConfigUpdates.ts | 25 ++++++------ 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/library/agent/realtime/connectToSSE.ts b/library/agent/realtime/connectToSSE.ts index 1ac3f1c3a..197989f84 100644 --- a/library/agent/realtime/connectToSSE.ts +++ b/library/agent/realtime/connectToSSE.ts @@ -26,6 +26,12 @@ export function connectToSSE({ const debugSSE = isDebuggingSSE(); + function logDebug(msg: string) { + if (debugSSE) { + logger.log(msg); + } + } + function connect() { if (currentRequest) { currentRequest.destroy(); @@ -34,9 +40,7 @@ export function connectToSSE({ const url = new URL(`${getRealtimeURL().toString()}api/runtime/stream`); - if (debugSSE) { - logger.log(`SSE connecting to ${url.toString()}`); - } + logDebug(`SSE connecting to ${url.toString()}`); const requestFn = url.protocol === "https:" ? requestHttps : requestHttp; @@ -52,18 +56,14 @@ export function connectToSSE({ }, (response) => { if (response.statusCode !== 200) { - logger.log( - `SSE connection failed with status ${response.statusCode}` - ); + logDebug(`SSE connection failed with status ${response.statusCode}`); response.destroy(); scheduleReconnect(); return; } reconnectMs = INITIAL_RECONNECT_MS; - if (debugSSE) { - logger.log("SSE connected successfully"); - } + logDebug("SSE connected successfully"); const parser = createParser({ onEvent(event) { @@ -74,20 +74,18 @@ export function connectToSSE({ response.setEncoding("utf-8"); response.on("data", (chunk: string) => { - if (debugSSE) { - logger.log(`SSE received chunk: ${chunk.trimEnd()}`); - } + logDebug(`SSE received chunk: ${chunk.trimEnd()}`); parser.feed(chunk); }); response.on("end", () => { - logger.log("SSE connection closed by server, reconnecting"); + logDebug("SSE connection closed by server, reconnecting"); parser.reset(); scheduleReconnect(); }); response.on("error", (error) => { - logger.log(`SSE stream error: ${error.message}`); + logDebug(`SSE stream error: ${error.message}`); parser.reset(); scheduleReconnect(); }); @@ -98,17 +96,16 @@ export function connectToSSE({ req.on("socket", (socket) => { socket.setTimeout(READ_TIMEOUT_MS, () => { - if (debugSSE) { - logger.log("SSE read timeout, reconnecting"); - } + logDebug("SSE read timeout, reconnecting"); req.destroy(); scheduleReconnect(); }); + // Don't keep the process alive just for the SSE connection socket.unref(); }); req.on("error", (error) => { - logger.log(`SSE connection error: ${error.message}`); + logDebug(`SSE connection error: ${error.message}`); scheduleReconnect(); }); @@ -120,12 +117,11 @@ export function connectToSSE({ clearTimeout(reconnectTimer); } - if (debugSSE) { - logger.log(`SSE scheduling reconnect in ${reconnectMs}ms`); - } + logDebug(`SSE scheduling reconnect in ${reconnectMs}ms`); reconnectTimer = setTimeout(connect, reconnectMs); reconnectMs = Math.min(reconnectMs * 2, MAX_RECONNECT_MS); + // Don't keep the process alive just for the reconnect timer reconnectTimer.unref(); } diff --git a/library/agent/realtime/listenForConfigUpdates.ts b/library/agent/realtime/listenForConfigUpdates.ts index a2226be90..f3832320a 100644 --- a/library/agent/realtime/listenForConfigUpdates.ts +++ b/library/agent/realtime/listenForConfigUpdates.ts @@ -25,15 +25,20 @@ export function listenForConfigUpdates({ const validToken = token; const debugSSE = isDebuggingSSE(); + + function logDebug(msg: string) { + if (debugSSE) { + logger.log(msg); + } + } + let currentLastUpdatedAt = lastUpdatedAt; connectToSSE({ token, logger, onEvent(event) { - if (debugSSE) { - logger.log(`SSE event received: ${event.event}`); - } + logDebug(`SSE event received: ${event.event}`); if (event.event !== "config-updated") { return; } @@ -47,22 +52,18 @@ export function listenForConfigUpdates({ // If we can't parse the payload, fetch the config anyway } - if (debugSSE) { - logger.log("SSE config-updated event, fetching new config"); - } + logDebug("SSE config-updated event, fetching new config"); getConfig(validToken) .then((config) => { - if (debugSSE) { - logger.log( - `SSE config fetched, configUpdatedAt: ${config.configUpdatedAt}` - ); - } + logDebug( + `SSE config fetched, configUpdatedAt: ${config.configUpdatedAt}` + ); currentLastUpdatedAt = config.configUpdatedAt; onConfigUpdate(config); }) .catch((error) => { - logger.log( + logDebug( `Failed to fetch config after SSE event: ${error.message}` ); }); From b4423cd389b46d9262b1480cecd1ddc2cf52b35c Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Wed, 20 May 2026 10:10:21 +0200 Subject: [PATCH 20/47] Skip SSE when realtime endpoint is unreachable --- library/agent/Agent.ts | 23 +++++---- library/agent/realtime/probeRealtimeURL.ts | 44 +++++++++++++++++ library/agent/realtime/resolvePollingURL.ts | 52 --------------------- 3 files changed, 59 insertions(+), 60 deletions(-) create mode 100644 library/agent/realtime/probeRealtimeURL.ts delete mode 100644 library/agent/realtime/resolvePollingURL.ts diff --git a/library/agent/Agent.ts b/library/agent/Agent.ts index 32b23a31a..1f724fe88 100644 --- a/library/agent/Agent.ts +++ b/library/agent/Agent.ts @@ -40,7 +40,7 @@ import type { IdorProtectionConfig } from "./IdorProtectionConfig"; import { warnIfTsxIsUsed } from "../helpers/warnIfTsxIsUsed"; import { pollForChanges } from "./realtime/pollForChanges"; import { getRealtimeURL } from "./realtime/getRealtimeURL"; -import { resolvePollingURL } from "./realtime/resolvePollingURL"; +import { probeRealtimeURL } from "./realtime/probeRealtimeURL"; type WrappedPackage = { version: string; supported: boolean }; @@ -469,18 +469,25 @@ export class Agent { const lastUpdatedAt = this.serviceConfig.getLastUpdatedAt(); - listenForConfigUpdates({ - token: this.token, - logger: this.logger, - lastUpdatedAt, - onConfigUpdate, - }); + const { pollingURL, realtimeReachable } = await probeRealtimeURL( + this.token, + this.logger + ); + + if (realtimeReachable) { + listenForConfigUpdates({ + token: this.token, + logger: this.logger, + lastUpdatedAt, + onConfigUpdate, + }); + } pollForChanges({ token: this.token, logger: this.logger, lastUpdatedAt, - realtimeURL: await resolvePollingURL(this.token, this.logger), + realtimeURL: pollingURL, onConfigUpdate, }); } diff --git a/library/agent/realtime/probeRealtimeURL.ts b/library/agent/realtime/probeRealtimeURL.ts new file mode 100644 index 000000000..b059a5d18 --- /dev/null +++ b/library/agent/realtime/probeRealtimeURL.ts @@ -0,0 +1,44 @@ +import { fetch } from "../../helpers/fetch"; +import type { Token } from "../api/Token"; +import type { Logger } from "../logger/Logger"; +import { getRealtimeURL } from "./getRealtimeURL"; + +type RealtimeProbeResult = { + pollingURL: URL; + realtimeReachable: boolean; +}; + +export async function probeRealtimeURL( + token: Token, + logger: Logger +): Promise { + const realtimeURL = getRealtimeURL(); + + if (process.env.AIKIDO_REALTIME_ENDPOINT) { + return { pollingURL: realtimeURL, realtimeReachable: true }; + } + + const configURL = new URL(`${realtimeURL.toString()}config`); + + try { + await fetch({ + url: configURL, + method: "GET", + headers: { + Authorization: token.asString(), + }, + timeoutInMS: 5000, + }); + + return { pollingURL: realtimeURL, realtimeReachable: true }; + } catch { + logger.log( + `Can't reach ${realtimeURL.hostname}, make sure it's in your outbound firewall allowlist. Realtime config updates won't be available, switched to polling.` + ); + + return { + pollingURL: new URL("https://runtime.aikido.dev"), + realtimeReachable: false, + }; + } +} diff --git a/library/agent/realtime/resolvePollingURL.ts b/library/agent/realtime/resolvePollingURL.ts deleted file mode 100644 index cfaaa3a6a..000000000 --- a/library/agent/realtime/resolvePollingURL.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { fetch } from "../../helpers/fetch"; -import type { Token } from "../api/Token"; -import type { Logger } from "../logger/Logger"; -import { getRealtimeURL } from "./getRealtimeURL"; - -const MAX_RETRIES = 3; - -async function probe(url: URL, token: Token): Promise { - try { - await fetch({ - url, - method: "GET", - headers: { - Authorization: token.asString(), - }, - timeoutInMS: 5000, - }); - - return true; - } catch { - return false; - } -} - -export async function resolvePollingURL( - token: Token, - logger: Logger -): Promise { - const realtimeURL = getRealtimeURL(); - - if (process.env.AIKIDO_REALTIME_ENDPOINT) { - return realtimeURL; - } - - const configURL = new URL(`${realtimeURL.toString()}config`); - let backoffMs = 1000; - - for (let i = 0; i < MAX_RETRIES; i++) { - if (await probe(configURL, token)) { - return realtimeURL; - } - - await new Promise((resolve) => setTimeout(resolve, backoffMs)); - backoffMs *= 2; - } - - logger.log( - `Can't reach ${realtimeURL.hostname}, make sure it's in your outbound firewall allowlist. Realtime config updates won't be available, switched to polling.` - ); - - return new URL("https://runtime.aikido.dev"); -} From 4c5024765e81ef1017f8ca9815f9428e5e113765 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Wed, 20 May 2026 10:13:57 +0200 Subject: [PATCH 21/47] Fix mock SSE event data to match zen-realtime --- end2end/server/src/handlers/stream.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/end2end/server/src/handlers/stream.ts b/end2end/server/src/handlers/stream.ts index a701a4928..a9b436c1b 100644 --- a/end2end/server/src/handlers/stream.ts +++ b/end2end/server/src/handlers/stream.ts @@ -17,7 +17,8 @@ export function stream(req: ZenRequest, res: Response) { function sendConfig() { const config = getAppConfig(app); - res.write(`event: config-updated\ndata: ${JSON.stringify(config)}\n\n`); + const data = { serviceId: app.id, configUpdatedAt: config.configUpdatedAt }; + res.write(`event: config-updated\ndata: ${JSON.stringify(data)}\n\n`); } sendConfig(); From 682918e3018c1c1030823de131b75cab0fe88f72 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Wed, 20 May 2026 10:21:56 +0200 Subject: [PATCH 22/47] Add SSE reconnect e2e test --- end2end/server/app.ts | 3 +- end2end/server/src/handlers/stream.ts | 24 +++++ .../realtime-config-updates.test.mjs | 96 +++++++++++++++++-- 3 files changed, 112 insertions(+), 11 deletions(-) diff --git a/end2end/server/app.ts b/end2end/server/app.ts index 4aa6cf638..1c66cd32f 100644 --- a/end2end/server/app.ts +++ b/end2end/server/app.ts @@ -10,7 +10,7 @@ import { updateConfig } from "./src/handlers/updateConfig.ts"; import { lists } from "./src/handlers/lists.ts"; import { updateIPLists } from "./src/handlers/updateLists.ts"; import { realtimeConfig } from "./src/handlers/realtimeConfig.ts"; -import { stream } from "./src/handlers/stream.ts"; +import { stream, disconnectStreams } from "./src/handlers/stream.ts"; const app = express(); app.set("trust proxy", false); @@ -26,6 +26,7 @@ app.post("/api/runtime/config", checkToken, updateConfig); // Realtime polling endpoint app.get("/config", checkToken, realtimeConfig); app.get("/api/runtime/stream", checkToken, stream); +app.post("/api/runtime/stream/disconnect", checkToken, disconnectStreams); app.get("/api/runtime/events", checkToken, listEvents); app.post("/api/runtime/events", checkToken, captureEvent); diff --git a/end2end/server/src/handlers/stream.ts b/end2end/server/src/handlers/stream.ts index a9b436c1b..d9a2ae560 100644 --- a/end2end/server/src/handlers/stream.ts +++ b/end2end/server/src/handlers/stream.ts @@ -2,6 +2,8 @@ import type { Response } from "express"; import { getAppConfig, configEvents } from "../zen/config.ts"; import type { ZenRequest } from "../types.ts"; +const connections = new Map>(); + export function stream(req: ZenRequest, res: Response) { if (!req.zenApp) { throw new Error("App is missing"); @@ -9,6 +11,11 @@ export function stream(req: ZenRequest, res: Response) { const app = req.zenApp; + if (!connections.has(app.id)) { + connections.set(app.id, new Set()); + } + connections.get(app.id)!.add(res); + res.writeHead(200, { "Content-Type": "text/event-stream", "Cache-Control": "no-cache", @@ -31,7 +38,24 @@ export function stream(req: ZenRequest, res: Response) { }, 30_000); req.on("close", () => { + connections.get(app.id)?.delete(res); configEvents.off(eventName, sendConfig); clearInterval(ping); }); } + +export function disconnectStreams(req: ZenRequest, res: Response) { + if (!req.zenApp) { + throw new Error("App is missing"); + } + + const appConnections = connections.get(req.zenApp.id); + if (appConnections) { + for (const conn of appConnections) { + conn.end(); + } + appConnections.clear(); + } + + res.json({ ok: true }); +} diff --git a/end2end/tests-new/realtime-config-updates.test.mjs b/end2end/tests-new/realtime-config-updates.test.mjs index a91649b40..da8673504 100644 --- a/end2end/tests-new/realtime-config-updates.test.mjs +++ b/end2end/tests-new/realtime-config-updates.test.mjs @@ -1,7 +1,7 @@ import { spawn } from "child_process"; import { resolve } from "path"; import { test } from "node:test"; -import { equal, fail } from "node:assert"; +import { equal, match, fail } from "node:assert"; import { getRandomPort } from "./utils/get-port.mjs"; import { timeout } from "./utils/timeout.mjs"; @@ -11,16 +11,9 @@ const pathToAppDir = resolve( ); const testServerUrl = "http://localhost:5874"; -const port = await getRandomPort(); -test("it picks up blocked IP via SSE config update", async () => { - const response = await fetch(`${testServerUrl}/api/runtime/apps`, { - method: "POST", - }); - const body = await response.json(); - const token = body.token; - - const server = spawn( +function spawnApp(token, port) { + return spawn( `node`, [ "--require", @@ -37,10 +30,22 @@ test("it picks up blocked IP via SSE config update", async () => { AIKIDO_ENDPOINT: testServerUrl, AIKIDO_REALTIME_ENDPOINT: testServerUrl, AIKIDO_DEBUG: "true", + AIKIDO_DEBUG_SSE: "true", AIKIDO_BLOCK: "true", }, } ); +} + +test("it picks up blocked IP via SSE config update", async () => { + const response = await fetch(`${testServerUrl}/api/runtime/apps`, { + method: "POST", + }); + const body = await response.json(); + const token = body.token; + const port = await getRandomPort(); + + const server = spawnApp(token, port); try { server.on("error", (err) => { @@ -94,3 +99,74 @@ test("it picks up blocked IP via SSE config update", async () => { server.kill(); } }); + +test("it reconnects SSE after server disconnects", async () => { + const response = await fetch(`${testServerUrl}/api/runtime/apps`, { + method: "POST", + }); + const body = await response.json(); + const token = body.token; + const port = await getRandomPort(); + + const server = spawnApp(token, port); + + try { + server.on("error", (err) => { + fail(err); + }); + + let stdout = ""; + server.stdout.on("data", (data) => { + stdout += data.toString(); + }); + + let stderr = ""; + server.stderr.on("data", (data) => { + stderr += data.toString(); + }); + + // Wait for the server to start and SSE to connect + await timeout(3000); + match(stdout, /SSE connected successfully/); + + // Disconnect SSE from the server side + await fetch(`${testServerUrl}/api/runtime/stream/disconnect`, { + method: "POST", + headers: { Authorization: token }, + }); + + // Wait for reconnect (initial reconnect delay is 1s) + await timeout(3000); + match(stdout, /SSE connection closed by server, reconnecting/); + + // Verify SSE reconnected + const connectedCount = stdout.split("SSE connected successfully").length - 1; + equal(connectedCount >= 2, true); + + // Block IP 9.8.7.6 after reconnect to verify the new connection works + await fetch(`${testServerUrl}/api/runtime/firewall/lists`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: token, + }, + body: JSON.stringify({ + blockedIPAddresses: ["9.8.7.6"], + }), + }); + + // Wait for SSE config-updated event to propagate + await timeout(2000); + + // Verify the blocked IP is picked up via the reconnected SSE + const blocked = await fetch(`http://127.0.0.1:${port}/`, { + headers: { "x-forwarded-for": "9.8.7.6" }, + signal: AbortSignal.timeout(5000), + }); + equal(blocked.status, 403); + } catch (err) { + fail(err); + } finally { + server.kill(); + } +}); From 1227246b7821618f64959c2fdc2858083c3ccc34 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Wed, 20 May 2026 10:26:28 +0200 Subject: [PATCH 23/47] Log and ignore invalid SSE config-updated payloads --- library/agent/realtime/listenForConfigUpdates.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/library/agent/realtime/listenForConfigUpdates.ts b/library/agent/realtime/listenForConfigUpdates.ts index f3832320a..62408a7ad 100644 --- a/library/agent/realtime/listenForConfigUpdates.ts +++ b/library/agent/realtime/listenForConfigUpdates.ts @@ -49,7 +49,8 @@ export function listenForConfigUpdates({ return; } } catch { - // If we can't parse the payload, fetch the config anyway + logDebug(`SSE config-updated event has invalid payload: ${event.data}`); + return; } logDebug("SSE config-updated event, fetching new config"); From 7703deadf75848852bab84682c9f343b4301f59c Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Wed, 20 May 2026 10:29:56 +0200 Subject: [PATCH 24/47] Format code --- end2end/tests-new/realtime-config-updates.test.mjs | 3 ++- library/agent/realtime/listenForConfigUpdates.ts | 4 +--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/end2end/tests-new/realtime-config-updates.test.mjs b/end2end/tests-new/realtime-config-updates.test.mjs index da8673504..3cb6f17d3 100644 --- a/end2end/tests-new/realtime-config-updates.test.mjs +++ b/end2end/tests-new/realtime-config-updates.test.mjs @@ -140,7 +140,8 @@ test("it reconnects SSE after server disconnects", async () => { match(stdout, /SSE connection closed by server, reconnecting/); // Verify SSE reconnected - const connectedCount = stdout.split("SSE connected successfully").length - 1; + const connectedCount = + stdout.split("SSE connected successfully").length - 1; equal(connectedCount >= 2, true); // Block IP 9.8.7.6 after reconnect to verify the new connection works diff --git a/library/agent/realtime/listenForConfigUpdates.ts b/library/agent/realtime/listenForConfigUpdates.ts index 62408a7ad..807479ac0 100644 --- a/library/agent/realtime/listenForConfigUpdates.ts +++ b/library/agent/realtime/listenForConfigUpdates.ts @@ -64,9 +64,7 @@ export function listenForConfigUpdates({ onConfigUpdate(config); }) .catch((error) => { - logDebug( - `Failed to fetch config after SSE event: ${error.message}` - ); + logDebug(`Failed to fetch config after SSE event: ${error.message}`); }); }, }); From 86e3e74bfad4c89e2150a3205adf55d6aa8e259f Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Wed, 20 May 2026 10:57:35 +0200 Subject: [PATCH 25/47] Remove token from sink tests to avoid probe hostname leak The new probeRealtimeURL call makes a real HTTP request to zen.aikido.dev when a token is present, which gets tracked as an outgoing hostname and breaks assertions. Same fix as the got test. --- library/sinks/Fetch.test.ts | 2 -- library/sinks/HTTPRequest.test.ts | 2 -- library/sinks/Undici.tests.ts | 3 +-- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/library/sinks/Fetch.test.ts b/library/sinks/Fetch.test.ts index 3b312660d..ee859b97c 100644 --- a/library/sinks/Fetch.test.ts +++ b/library/sinks/Fetch.test.ts @@ -1,6 +1,5 @@ import * as t from "tap"; import { ReportingAPIForTesting } from "../agent/api/ReportingAPIForTesting"; -import { Token } from "../agent/api/Token"; import { Context, runWithContext } from "../agent/Context"; import { wrap } from "../helpers/wrap"; import { Fetch } from "./Fetch"; @@ -83,7 +82,6 @@ t.test( async (t) => { const api = new ReportingAPIForTesting(); const agent = createTestAgent({ - token: new Token("123"), api, }); diff --git a/library/sinks/HTTPRequest.test.ts b/library/sinks/HTTPRequest.test.ts index b7869304f..6d0f7f28c 100644 --- a/library/sinks/HTTPRequest.test.ts +++ b/library/sinks/HTTPRequest.test.ts @@ -1,6 +1,5 @@ import * as dns from "dns"; import * as t from "tap"; -import { Token } from "../agent/api/Token"; import { Context, runWithContext } from "../agent/Context"; import { wrap } from "../helpers/wrap"; import { HTTPRequest } from "./HTTPRequest"; @@ -61,7 +60,6 @@ t.setTimeout(60 * 1000); const api = new ReportingAPIForTesting(); const agent = createTestAgent({ - token: new Token("123"), api, }); agent.start([new HTTPRequest()]); diff --git a/library/sinks/Undici.tests.ts b/library/sinks/Undici.tests.ts index 671b2e7f3..4c6a6919e 100644 --- a/library/sinks/Undici.tests.ts +++ b/library/sinks/Undici.tests.ts @@ -1,6 +1,6 @@ import * as t from "tap"; import { ReportingAPIForTesting } from "../agent/api/ReportingAPIForTesting"; -import { Token } from "../agent/api/Token"; + import { Context, runWithContext } from "../agent/Context"; import { LoggerForTesting } from "../agent/logger/LoggerForTesting"; import { startTestAgent } from "../helpers/startTestAgent"; @@ -60,7 +60,6 @@ export async function createUndiciTests(undiciPkgName: string, port: number) { const agent = startTestAgent({ api, logger, - token: new Token("123"), wrappers: [new Undici()], rewrite: { undici: undiciPkgName, From 85cc33ab7995abe193e61ea3fe869dcc4b2fda0a Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Wed, 20 May 2026 12:15:59 +0200 Subject: [PATCH 26/47] Drain probeRealtimeURL fetch timeout in Agent tests --- library/agent/Agent.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/library/agent/Agent.test.ts b/library/agent/Agent.test.ts index 8eb6354bd..394027dc5 100644 --- a/library/agent/Agent.test.ts +++ b/library/agent/Agent.test.ts @@ -459,6 +459,8 @@ t.test( // After a minute, we'll see that the dashboard didn't receive any stats yet // And then send a heartbeat clock.tick(60 * 1000); + // Extra nextAsync to drain the fetch timeout from probeRealtimeURL + await clock.nextAsync(); await clock.nextAsync(); t.match(api.getEvents(), [ { @@ -526,6 +528,7 @@ t.test( // But the stats is still empty, so we won't send a heartbeat clock.tick(60 * 1000); await clock.nextAsync(); + await clock.nextAsync(); t.match(api.getEvents(), [ { type: "started", @@ -586,6 +589,7 @@ t.test("it sends heartbeat when reached max timings", async () => { // After 30 seconds, the first heartbeat should be sent clock.tick(30 * 1000); await clock.nextAsync(); + await clock.nextAsync(); t.match(api.getEvents(), [ { @@ -734,6 +738,7 @@ t.test("unable to prevent prototype pollution", async () => { clock.tick(1000 * 60 * 30); await clock.nextAsync(); + await clock.nextAsync(); t.same(api.getEvents().length, 2); const [_, heartbeat] = api.getEvents(); From a88ae85deebc9a01bafbbd66a71ac232196d063a Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Wed, 20 May 2026 12:23:18 +0200 Subject: [PATCH 27/47] Allow localhost in outbound blocking e2e test --- end2end/tests/hono-xml-outbound.test.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/end2end/tests/hono-xml-outbound.test.js b/end2end/tests/hono-xml-outbound.test.js index 400350bc5..66f39e5e3 100644 --- a/end2end/tests/hono-xml-outbound.test.js +++ b/end2end/tests/hono-xml-outbound.test.js @@ -132,6 +132,8 @@ t.test("blockNewOutgoingRequests is true", (t) => { domains: [ { hostname: "ssrf-redirects.testssandbox.com", mode: "block" }, { hostname: "aikido.dev", mode: "allow" }, + // Otherwise we cannot communicate with the mock server + { hostname: "localhost", mode: "allow" }, ], }), }); From abddb97881bba485fdfe0f952c05510357b264f7 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Wed, 20 May 2026 12:33:54 +0200 Subject: [PATCH 28/47] Account for SSE connection in heartbeat hostname hits --- end2end/tests-new/heartbeat.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/end2end/tests-new/heartbeat.test.mjs b/end2end/tests-new/heartbeat.test.mjs index 1da88a6c2..548dee847 100644 --- a/end2end/tests-new/heartbeat.test.mjs +++ b/end2end/tests-new/heartbeat.test.mjs @@ -90,7 +90,7 @@ test("It reports own http requests in heartbeat events", async () => { { hostname: "localhost", port: 5874, - hits: 3, + hits: 4, }, ], agent: { From 55f9c2afe5302f88e9c051a50b43919ce77f0774 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Wed, 20 May 2026 12:43:49 +0200 Subject: [PATCH 29/47] Add jitter and stable-connection backoff to SSE reconnect --- .../realtime-config-updates.test.mjs | 4 ++-- library/agent/realtime/connectToSSE.ts | 20 +++++++++++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/end2end/tests-new/realtime-config-updates.test.mjs b/end2end/tests-new/realtime-config-updates.test.mjs index 3cb6f17d3..81fc8a8eb 100644 --- a/end2end/tests-new/realtime-config-updates.test.mjs +++ b/end2end/tests-new/realtime-config-updates.test.mjs @@ -135,8 +135,8 @@ test("it reconnects SSE after server disconnects", async () => { headers: { Authorization: token }, }); - // Wait for reconnect (initial reconnect delay is 1s) - await timeout(3000); + // Wait for reconnect (initial reconnect delay is 5s + jitter) + await timeout(9000); match(stdout, /SSE connection closed by server, reconnecting/); // Verify SSE reconnected diff --git a/library/agent/realtime/connectToSSE.ts b/library/agent/realtime/connectToSSE.ts index 197989f84..322a0586b 100644 --- a/library/agent/realtime/connectToSSE.ts +++ b/library/agent/realtime/connectToSSE.ts @@ -7,8 +7,9 @@ import { Token } from "../api/Token"; import { Logger } from "../logger/Logger"; import { getRealtimeURL } from "./getRealtimeURL"; -const INITIAL_RECONNECT_MS = 1000; +const INITIAL_RECONNECT_MS = 5000; const MAX_RECONNECT_MS = 60 * 1000; +const STABLE_CONNECTION_MS = 30 * 1000; const READ_TIMEOUT_MS = 70 * 1000; export function connectToSSE({ @@ -62,7 +63,7 @@ export function connectToSSE({ return; } - reconnectMs = INITIAL_RECONNECT_MS; + const connectedAt = Date.now(); logDebug("SSE connected successfully"); const parser = createParser({ @@ -81,12 +82,14 @@ export function connectToSSE({ response.on("end", () => { logDebug("SSE connection closed by server, reconnecting"); parser.reset(); + resetBackoffIfStable(connectedAt); scheduleReconnect(); }); response.on("error", (error) => { logDebug(`SSE stream error: ${error.message}`); parser.reset(); + resetBackoffIfStable(connectedAt); scheduleReconnect(); }); } @@ -112,14 +115,23 @@ export function connectToSSE({ req.end(); } + function resetBackoffIfStable(connectedAt: number) { + if (Date.now() - connectedAt >= STABLE_CONNECTION_MS) { + reconnectMs = INITIAL_RECONNECT_MS; + } + } + function scheduleReconnect() { if (reconnectTimer) { clearTimeout(reconnectTimer); } - logDebug(`SSE scheduling reconnect in ${reconnectMs}ms`); + const jitter = Math.random() * (reconnectMs / 2); + const delay = reconnectMs + jitter; + + logDebug(`SSE scheduling reconnect in ${Math.round(delay)}ms`); - reconnectTimer = setTimeout(connect, reconnectMs); + reconnectTimer = setTimeout(connect, delay); reconnectMs = Math.min(reconnectMs * 2, MAX_RECONNECT_MS); // Don't keep the process alive just for the reconnect timer reconnectTimer.unref(); From b88692c71227e1ed3b4378e436cbe188e2760b28 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Wed, 20 May 2026 13:04:00 +0200 Subject: [PATCH 30/47] Stop SSE reconnect on 401 and 403 --- end2end/server/app.ts | 2 + end2end/server/src/handlers/deleteApp.ts | 14 +++++ end2end/server/src/handlers/stream.ts | 15 +++-- end2end/server/src/zen/apps.ts | 6 +- .../realtime-config-updates.test.mjs | 56 ++++++++++++++++++- library/agent/realtime/connectToSSE.ts | 12 ++++ 6 files changed, 95 insertions(+), 10 deletions(-) create mode 100644 end2end/server/src/handlers/deleteApp.ts diff --git a/end2end/server/app.ts b/end2end/server/app.ts index 1c66cd32f..332c17638 100644 --- a/end2end/server/app.ts +++ b/end2end/server/app.ts @@ -11,6 +11,7 @@ import { lists } from "./src/handlers/lists.ts"; import { updateIPLists } from "./src/handlers/updateLists.ts"; import { realtimeConfig } from "./src/handlers/realtimeConfig.ts"; import { stream, disconnectStreams } from "./src/handlers/stream.ts"; +import { deleteApp } from "./src/handlers/deleteApp.ts"; const app = express(); app.set("trust proxy", false); @@ -35,6 +36,7 @@ app.get("/api/runtime/firewall/lists", checkToken, lists); app.post("/api/runtime/firewall/lists", checkToken, updateIPLists); app.post("/api/runtime/apps", createApp); +app.delete("/api/runtime/apps", checkToken, deleteApp); app.listen(port, () => { console.log(`Server is running on port ${port}`); diff --git a/end2end/server/src/handlers/deleteApp.ts b/end2end/server/src/handlers/deleteApp.ts new file mode 100644 index 000000000..5e39657ab --- /dev/null +++ b/end2end/server/src/handlers/deleteApp.ts @@ -0,0 +1,14 @@ +import type { Response } from "express"; +import { removeApp } from "../zen/apps.ts"; +import { closeStreams } from "./stream.ts"; +import type { ZenRequest } from "../types.ts"; + +export function deleteApp(req: ZenRequest, res: Response) { + if (!req.zenApp) { + throw new Error("App is missing"); + } + + removeApp(req.zenApp); + closeStreams(req.zenApp.id); + res.json({ ok: true }); +} diff --git a/end2end/server/src/handlers/stream.ts b/end2end/server/src/handlers/stream.ts index d9a2ae560..90a9873f1 100644 --- a/end2end/server/src/handlers/stream.ts +++ b/end2end/server/src/handlers/stream.ts @@ -44,18 +44,21 @@ export function stream(req: ZenRequest, res: Response) { }); } -export function disconnectStreams(req: ZenRequest, res: Response) { - if (!req.zenApp) { - throw new Error("App is missing"); - } - - const appConnections = connections.get(req.zenApp.id); +export function closeStreams(appId: number) { + const appConnections = connections.get(appId); if (appConnections) { for (const conn of appConnections) { conn.end(); } appConnections.clear(); } +} + +export function disconnectStreams(req: ZenRequest, res: Response) { + if (!req.zenApp) { + throw new Error("App is missing"); + } + closeStreams(req.zenApp.id); res.json({ ok: true }); } diff --git a/end2end/server/src/zen/apps.ts b/end2end/server/src/zen/apps.ts index e11e2cd99..a95680e0f 100644 --- a/end2end/server/src/zen/apps.ts +++ b/end2end/server/src/zen/apps.ts @@ -6,7 +6,7 @@ export type App = { configUpdatedAt: number; }; -const apps: App[] = []; +let apps: App[] = []; let id = 1; export function createApp(): string { @@ -20,6 +20,10 @@ export function createApp(): string { return token; } +export function removeApp(app: App): void { + apps = apps.filter((a) => a.id !== app.id); +} + export function getByToken(token: string): App | undefined { return apps.find((app) => { if (app.token.length !== token.length) { diff --git a/end2end/tests-new/realtime-config-updates.test.mjs b/end2end/tests-new/realtime-config-updates.test.mjs index 81fc8a8eb..54bf0a462 100644 --- a/end2end/tests-new/realtime-config-updates.test.mjs +++ b/end2end/tests-new/realtime-config-updates.test.mjs @@ -1,7 +1,7 @@ import { spawn } from "child_process"; import { resolve } from "path"; import { test } from "node:test"; -import { equal, match, fail } from "node:assert"; +import { equal, doesNotMatch, match, fail } from "node:assert"; import { getRandomPort } from "./utils/get-port.mjs"; import { timeout } from "./utils/timeout.mjs"; @@ -135,8 +135,8 @@ test("it reconnects SSE after server disconnects", async () => { headers: { Authorization: token }, }); - // Wait for reconnect (initial reconnect delay is 5s + jitter) - await timeout(9000); + // Wait for reconnect (initial reconnect delay is 5s + jitter up to 7.5s) + await timeout(10000); match(stdout, /SSE connection closed by server, reconnecting/); // Verify SSE reconnected @@ -171,3 +171,53 @@ test("it reconnects SSE after server disconnects", async () => { server.kill(); } }); + +test("it stops SSE reconnect on 401", async () => { + const response = await fetch(`${testServerUrl}/api/runtime/apps`, { + method: "POST", + }); + const body = await response.json(); + const token = body.token; + const port = await getRandomPort(); + + const server = spawnApp(token, port); + + try { + server.on("error", (err) => { + fail(err); + }); + + let stdout = ""; + server.stdout.on("data", (data) => { + stdout += data.toString(); + }); + + let stderr = ""; + server.stderr.on("data", (data) => { + stderr += data.toString(); + }); + + // Wait for the server to start and SSE to connect + await timeout(3000); + match(stdout, /SSE connected successfully/); + + // Revoke the token and disconnect SSE so it tries to reconnect with 401 + await fetch(`${testServerUrl}/api/runtime/apps`, { + method: "DELETE", + headers: { Authorization: token }, + }); + + // Wait for reconnect attempts (may take multiple due to backoff + jitter) + await timeout(15000); + + match(stdout, /SSE connection rejected with status 401, stopping/); + // Should not schedule a reconnect after 401 + const rejectedIndex = stdout.indexOf("SSE connection rejected"); + const afterRejected = stdout.slice(rejectedIndex); + doesNotMatch(afterRejected, /SSE scheduling reconnect/); + } catch (err) { + fail(err); + } finally { + server.kill(); + } +}); diff --git a/library/agent/realtime/connectToSSE.ts b/library/agent/realtime/connectToSSE.ts index 322a0586b..e6abbe9a0 100644 --- a/library/agent/realtime/connectToSSE.ts +++ b/library/agent/realtime/connectToSSE.ts @@ -56,6 +56,14 @@ export function connectToSSE({ }, }, (response) => { + if (response.statusCode === 401 || response.statusCode === 403) { + logger.log( + `SSE connection rejected with status ${response.statusCode}, stopping` + ); + response.destroy(); + return; + } + if (response.statusCode !== 200) { logDebug(`SSE connection failed with status ${response.statusCode}`); response.destroy(); @@ -99,6 +107,10 @@ export function connectToSSE({ req.on("socket", (socket) => { socket.setTimeout(READ_TIMEOUT_MS, () => { + // Timeout can fire after response end/error already triggered reconnect + if (socket.destroyed) { + return; + } logDebug("SSE read timeout, reconnecting"); req.destroy(); scheduleReconnect(); From 4397a763c05c63ea71660acc75ac6cf91ebb43cf Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Wed, 20 May 2026 13:05:07 +0200 Subject: [PATCH 31/47] Allow localhost in outbound blocking e2e test --- end2end/tests-new/hono-pg-esm-outbound.test.mjs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/end2end/tests-new/hono-pg-esm-outbound.test.mjs b/end2end/tests-new/hono-pg-esm-outbound.test.mjs index c63ad5556..a8d8fb77a 100644 --- a/end2end/tests-new/hono-pg-esm-outbound.test.mjs +++ b/end2end/tests-new/hono-pg-esm-outbound.test.mjs @@ -132,6 +132,8 @@ test("blockNewOutgoingRequests is true", async () => { domains: [ { hostname: "ssrf-redirects.testssandbox.com", mode: "block" }, { hostname: "aikido.dev", mode: "allow" }, + // Otherwise we cannot communicate with the mock server + { hostname: "localhost", mode: "allow" }, ], }), }); From 8924af42e95c65ed528566772e0d0a6967e8c04e Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Wed, 20 May 2026 13:52:32 +0200 Subject: [PATCH 32/47] Add token to Fetch test to fix attack event assertion Without a token, Agent.onDetectedAttack skips reporting to the API, so api.getEvents() returns nothing and the test fails. --- library/sinks/Fetch.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/library/sinks/Fetch.test.ts b/library/sinks/Fetch.test.ts index ee859b97c..3b312660d 100644 --- a/library/sinks/Fetch.test.ts +++ b/library/sinks/Fetch.test.ts @@ -1,5 +1,6 @@ import * as t from "tap"; import { ReportingAPIForTesting } from "../agent/api/ReportingAPIForTesting"; +import { Token } from "../agent/api/Token"; import { Context, runWithContext } from "../agent/Context"; import { wrap } from "../helpers/wrap"; import { Fetch } from "./Fetch"; @@ -82,6 +83,7 @@ t.test( async (t) => { const api = new ReportingAPIForTesting(); const agent = createTestAgent({ + token: new Token("123"), api, }); From d4d0c9a9f8eea70042fa5eabbdd23fc957ae36b8 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Wed, 20 May 2026 15:10:57 +0200 Subject: [PATCH 33/47] Account for probe hostname in Fetch and Undici tests --- library/sinks/Fetch.test.ts | 1 + library/sinks/Undici.tests.ts | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/library/sinks/Fetch.test.ts b/library/sinks/Fetch.test.ts index 3b312660d..710c8c800 100644 --- a/library/sinks/Fetch.test.ts +++ b/library/sinks/Fetch.test.ts @@ -95,6 +95,7 @@ t.test( t.same(agent.getHostnames().asArray(), [ { hostname: "app.aikido.dev", port: 80, hits: 1 }, + { hostname: "zen.aikido.dev", port: 443, hits: 1 }, ]); agent.getHostnames().clear(); diff --git a/library/sinks/Undici.tests.ts b/library/sinks/Undici.tests.ts index 4c6a6919e..3c13faec1 100644 --- a/library/sinks/Undici.tests.ts +++ b/library/sinks/Undici.tests.ts @@ -1,6 +1,6 @@ import * as t from "tap"; import { ReportingAPIForTesting } from "../agent/api/ReportingAPIForTesting"; - +import { Token } from "../agent/api/Token"; import { Context, runWithContext } from "../agent/Context"; import { LoggerForTesting } from "../agent/logger/LoggerForTesting"; import { startTestAgent } from "../helpers/startTestAgent"; @@ -60,6 +60,7 @@ export async function createUndiciTests(undiciPkgName: string, port: number) { const agent = startTestAgent({ api, logger, + token: new Token("123"), wrappers: [new Undici()], rewrite: { undici: undiciPkgName, @@ -77,6 +78,11 @@ export async function createUndiciTests(undiciPkgName: string, port: number) { port: 443, hits: 1, }, + { + hostname: "zen.aikido.dev", + port: 443, + hits: 1, + }, ]); agent.getHostnames().clear(); From 708666f85a949b89061b1766fe1316e54bfdf963 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Wed, 20 May 2026 15:34:17 +0200 Subject: [PATCH 34/47] Sort hostnames in Undici test to fix ordering flake --- library/sinks/Undici.tests.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/library/sinks/Undici.tests.ts b/library/sinks/Undici.tests.ts index 3c13faec1..b63380f50 100644 --- a/library/sinks/Undici.tests.ts +++ b/library/sinks/Undici.tests.ts @@ -7,6 +7,12 @@ import { startTestAgent } from "../helpers/startTestAgent"; import { getMajorNodeVersion } from "../helpers/getNodeVersion"; import { Undici } from "./Undici"; +type HostnameEntry = { hostname: string; port: number; hits: number }; + +function sortedByHostname(entries: HostnameEntry[]) { + return entries.sort((a, b) => a.hostname.localeCompare(b.hostname)); +} + // Undici tests are split up because sockets are re-used for the same hostname // See Undici.tests.ts and Undici2.tests.ts // Async needed because `require(...)` is translated to `await import(..)` when running tests in ESM mode @@ -72,7 +78,7 @@ export async function createUndiciTests(undiciPkgName: string, port: number) { ) as typeof import("undici-v6"); await request("https://ssrf-redirects.testssandbox.com"); - t.same(agent.getHostnames().asArray(), [ + t.same(sortedByHostname(agent.getHostnames().asArray()), [ { hostname: "ssrf-redirects.testssandbox.com", port: 443, From 6e49962dc8be5882392c4249a9f68fccd8bf8c64 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Wed, 20 May 2026 15:39:14 +0200 Subject: [PATCH 35/47] Restore token in HTTPRequest test for attack event assertion --- library/sinks/HTTPRequest.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/library/sinks/HTTPRequest.test.ts b/library/sinks/HTTPRequest.test.ts index 6d0f7f28c..b7869304f 100644 --- a/library/sinks/HTTPRequest.test.ts +++ b/library/sinks/HTTPRequest.test.ts @@ -1,5 +1,6 @@ import * as dns from "dns"; import * as t from "tap"; +import { Token } from "../agent/api/Token"; import { Context, runWithContext } from "../agent/Context"; import { wrap } from "../helpers/wrap"; import { HTTPRequest } from "./HTTPRequest"; @@ -60,6 +61,7 @@ t.setTimeout(60 * 1000); const api = new ReportingAPIForTesting(); const agent = createTestAgent({ + token: new Token("123"), api, }); agent.start([new HTTPRequest()]); From ed006ef44a283814012bd01ae65cde4df6e7e248 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Wed, 20 May 2026 15:55:15 +0200 Subject: [PATCH 36/47] Clear probe hostname before sink test assertions --- library/sinks/Fetch.test.ts | 2 +- library/sinks/HTTPRequest.test.ts | 1 + library/sinks/Undici.tests.ts | 16 ++++------------ 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/library/sinks/Fetch.test.ts b/library/sinks/Fetch.test.ts index 710c8c800..3484b44b8 100644 --- a/library/sinks/Fetch.test.ts +++ b/library/sinks/Fetch.test.ts @@ -89,13 +89,13 @@ t.test( agent.start([new Fetch()]); + agent.getHostnames().clear(); t.same(agent.getHostnames().asArray(), []); await fetch("http://app.aikido.dev"); t.same(agent.getHostnames().asArray(), [ { hostname: "app.aikido.dev", port: 80, hits: 1 }, - { hostname: "zen.aikido.dev", port: 443, hits: 1 }, ]); agent.getHostnames().clear(); diff --git a/library/sinks/HTTPRequest.test.ts b/library/sinks/HTTPRequest.test.ts index b7869304f..e3d6a95d0 100644 --- a/library/sinks/HTTPRequest.test.ts +++ b/library/sinks/HTTPRequest.test.ts @@ -71,6 +71,7 @@ const https = require("https") as typeof import("https"); const oldUrl = require("url"); t.test("it works", (t) => { + agent.getHostnames().clear(); t.same(agent.getHostnames().asArray(), []); runWithContext(createContext(), () => { diff --git a/library/sinks/Undici.tests.ts b/library/sinks/Undici.tests.ts index b63380f50..49fe73aaf 100644 --- a/library/sinks/Undici.tests.ts +++ b/library/sinks/Undici.tests.ts @@ -7,12 +7,6 @@ import { startTestAgent } from "../helpers/startTestAgent"; import { getMajorNodeVersion } from "../helpers/getNodeVersion"; import { Undici } from "./Undici"; -type HostnameEntry = { hostname: string; port: number; hits: number }; - -function sortedByHostname(entries: HostnameEntry[]) { - return entries.sort((a, b) => a.hostname.localeCompare(b.hostname)); -} - // Undici tests are split up because sockets are re-used for the same hostname // See Undici.tests.ts and Undici2.tests.ts // Async needed because `require(...)` is translated to `await import(..)` when running tests in ESM mode @@ -77,18 +71,16 @@ export async function createUndiciTests(undiciPkgName: string, port: number) { undiciPkgName ) as typeof import("undici-v6"); + agent.getHostnames().clear(); + t.same(agent.getHostnames().asArray(), []); + await request("https://ssrf-redirects.testssandbox.com"); - t.same(sortedByHostname(agent.getHostnames().asArray()), [ + t.same(agent.getHostnames().asArray(), [ { hostname: "ssrf-redirects.testssandbox.com", port: 443, hits: 1, }, - { - hostname: "zen.aikido.dev", - port: 443, - hits: 1, - }, ]); agent.getHostnames().clear(); From 5ae3353ac716c6dc033293ce8da6f7881d32b4e9 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Wed, 20 May 2026 16:25:10 +0200 Subject: [PATCH 37/47] Wait for realtime probe before sink test assertions --- library/sinks/Fetch.test.ts | 3 +++ library/sinks/Undici.tests.ts | 3 +++ 2 files changed, 6 insertions(+) diff --git a/library/sinks/Fetch.test.ts b/library/sinks/Fetch.test.ts index 3484b44b8..17a8b77d9 100644 --- a/library/sinks/Fetch.test.ts +++ b/library/sinks/Fetch.test.ts @@ -1,4 +1,5 @@ import * as t from "tap"; +import { setTimeout } from "timers/promises"; import { ReportingAPIForTesting } from "../agent/api/ReportingAPIForTesting"; import { Token } from "../agent/api/Token"; import { Context, runWithContext } from "../agent/Context"; @@ -89,6 +90,8 @@ t.test( agent.start([new Fetch()]); + // Let the realtime probe resolve before we start asserting + await setTimeout(500); agent.getHostnames().clear(); t.same(agent.getHostnames().asArray(), []); diff --git a/library/sinks/Undici.tests.ts b/library/sinks/Undici.tests.ts index 49fe73aaf..339ae16b0 100644 --- a/library/sinks/Undici.tests.ts +++ b/library/sinks/Undici.tests.ts @@ -1,4 +1,5 @@ import * as t from "tap"; +import { setTimeout } from "timers/promises"; import { ReportingAPIForTesting } from "../agent/api/ReportingAPIForTesting"; import { Token } from "../agent/api/Token"; import { Context, runWithContext } from "../agent/Context"; @@ -71,6 +72,8 @@ export async function createUndiciTests(undiciPkgName: string, port: number) { undiciPkgName ) as typeof import("undici-v6"); + // Let the realtime probe resolve before we start asserting + await setTimeout(500); agent.getHostnames().clear(); t.same(agent.getHostnames().asArray(), []); From 71839c97d6e8960f6f9646e8fe2311a6ecbae8ba Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Thu, 21 May 2026 10:15:12 +0200 Subject: [PATCH 38/47] Fix double backoff in SSE reconnect When a socket timeout fires, req.destroy() triggers the error handler which also calls scheduleReconnect(). Two calls per disconnect cause the backoff to double twice (5s->20s instead of 5s->10s). Remove scheduleReconnect() from the timeout handler (the error handler already does it) and add a guard so only the first call per connection advances the backoff. --- library/agent/realtime/connectToSSE.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/library/agent/realtime/connectToSSE.ts b/library/agent/realtime/connectToSSE.ts index e6abbe9a0..f39c19e5b 100644 --- a/library/agent/realtime/connectToSSE.ts +++ b/library/agent/realtime/connectToSSE.ts @@ -34,6 +34,8 @@ export function connectToSSE({ } function connect() { + reconnectScheduled = false; + if (currentRequest) { currentRequest.destroy(); currentRequest = null; @@ -107,13 +109,11 @@ export function connectToSSE({ req.on("socket", (socket) => { socket.setTimeout(READ_TIMEOUT_MS, () => { - // Timeout can fire after response end/error already triggered reconnect if (socket.destroyed) { return; } logDebug("SSE read timeout, reconnecting"); req.destroy(); - scheduleReconnect(); }); // Don't keep the process alive just for the SSE connection socket.unref(); @@ -133,7 +133,14 @@ export function connectToSSE({ } } + let reconnectScheduled = false; + function scheduleReconnect() { + if (reconnectScheduled) { + return; + } + reconnectScheduled = true; + if (reconnectTimer) { clearTimeout(reconnectTimer); } From 905bc15e28d7562e6519a9b56875e33596d52035 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Thu, 21 May 2026 10:43:40 +0200 Subject: [PATCH 39/47] Add unit tests for connectToSSE --- .../agent/realtime/connectToSSE.401.test.ts | 39 +++++++++++++ .../agent/realtime/connectToSSE.500.test.ts | 44 +++++++++++++++ .../agent/realtime/connectToSSE.auth.test.ts | 38 +++++++++++++ .../realtime/connectToSSE.connrefused.test.ts | 28 ++++++++++ .../realtime/connectToSSE.events.test.ts | 55 +++++++++++++++++++ .../agent/realtime/connectToSSE.ping.test.ts | 41 ++++++++++++++ .../realtime/connectToSSE.reconnect.test.ts | 46 ++++++++++++++++ 7 files changed, 291 insertions(+) create mode 100644 library/agent/realtime/connectToSSE.401.test.ts create mode 100644 library/agent/realtime/connectToSSE.500.test.ts create mode 100644 library/agent/realtime/connectToSSE.auth.test.ts create mode 100644 library/agent/realtime/connectToSSE.connrefused.test.ts create mode 100644 library/agent/realtime/connectToSSE.events.test.ts create mode 100644 library/agent/realtime/connectToSSE.ping.test.ts create mode 100644 library/agent/realtime/connectToSSE.reconnect.test.ts diff --git a/library/agent/realtime/connectToSSE.401.test.ts b/library/agent/realtime/connectToSSE.401.test.ts new file mode 100644 index 000000000..be6474cf0 --- /dev/null +++ b/library/agent/realtime/connectToSSE.401.test.ts @@ -0,0 +1,39 @@ +import * as t from "tap"; +import { createServer } from "http"; +import { Token } from "../api/Token"; +import { LoggerForTesting } from "../logger/LoggerForTesting"; +import { connectToSSE } from "./connectToSSE"; + +t.test("it stops reconnecting on 401", async (t) => { + let connectionCount = 0; + + const server = createServer((_req, res) => { + connectionCount++; + res.writeHead(401); + res.end(); + }); + + await new Promise((resolve) => server.listen(0, resolve)); + const port = (server.address() as { port: number }).port; + process.env.AIKIDO_REALTIME_ENDPOINT = `http://localhost:${port}/`; + + const logger = new LoggerForTesting(); + + try { + connectToSSE({ + token: new Token("bad-token"), + logger, + onEvent() {}, + }); + + await new Promise((r) => setTimeout(r, 500)); + + t.equal(connectionCount, 1); + t.match(logger.getMessages(), [ + /SSE connection rejected with status 401, stopping/, + ]); + } finally { + server.closeAllConnections(); + server.close(); + } +}); diff --git a/library/agent/realtime/connectToSSE.500.test.ts b/library/agent/realtime/connectToSSE.500.test.ts new file mode 100644 index 000000000..effae28e3 --- /dev/null +++ b/library/agent/realtime/connectToSSE.500.test.ts @@ -0,0 +1,44 @@ +import * as t from "tap"; +import { createServer } from "http"; +import { Token } from "../api/Token"; +import { LoggerForTesting } from "../logger/LoggerForTesting"; +import { connectToSSE } from "./connectToSSE"; + +t.test("it reconnects on non-200 status", async (t) => { + let connectionCount = 0; + + const server = createServer((_req, res) => { + connectionCount++; + if (connectionCount === 1) { + res.writeHead(500); + res.end(); + return; + } + res.writeHead(200, { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }); + res.write(": ping\n\n"); + }); + + await new Promise((resolve) => server.listen(0, resolve)); + const port = (server.address() as { port: number }).port; + process.env.AIKIDO_REALTIME_ENDPOINT = `http://localhost:${port}/`; + + try { + connectToSSE({ + token: new Token("test-token"), + logger: new LoggerForTesting(), + onEvent() {}, + }); + + // Wait for reconnect after 500 (initial delay 5s + up to 2.5s jitter) + await new Promise((r) => setTimeout(r, 8000)); + + t.equal(connectionCount, 2); + } finally { + server.closeAllConnections(); + server.close(); + } +}); diff --git a/library/agent/realtime/connectToSSE.auth.test.ts b/library/agent/realtime/connectToSSE.auth.test.ts new file mode 100644 index 000000000..9753e6923 --- /dev/null +++ b/library/agent/realtime/connectToSSE.auth.test.ts @@ -0,0 +1,38 @@ +import * as t from "tap"; +import { createServer } from "http"; +import { Token } from "../api/Token"; +import { LoggerForTesting } from "../logger/LoggerForTesting"; +import { connectToSSE } from "./connectToSSE"; + +t.test("it sends Authorization header", async (t) => { + let receivedAuth: string | undefined; + + const server = createServer((req, res) => { + receivedAuth = req.headers.authorization; + res.writeHead(200, { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }); + res.write(": ping\n\n"); + }); + + await new Promise((resolve) => server.listen(0, resolve)); + const port = (server.address() as { port: number }).port; + process.env.AIKIDO_REALTIME_ENDPOINT = `http://localhost:${port}/`; + + try { + connectToSSE({ + token: new Token("my-secret-token"), + logger: new LoggerForTesting(), + onEvent() {}, + }); + + await new Promise((r) => setTimeout(r, 200)); + + t.equal(receivedAuth, "my-secret-token"); + } finally { + server.closeAllConnections(); + server.close(); + } +}); diff --git a/library/agent/realtime/connectToSSE.connrefused.test.ts b/library/agent/realtime/connectToSSE.connrefused.test.ts new file mode 100644 index 000000000..48a79e78a --- /dev/null +++ b/library/agent/realtime/connectToSSE.connrefused.test.ts @@ -0,0 +1,28 @@ +import * as t from "tap"; +import { createServer } from "http"; +import { Token } from "../api/Token"; +import { LoggerForTesting } from "../logger/LoggerForTesting"; +import { connectToSSE } from "./connectToSSE"; + +t.test("it handles connection refused", async (t) => { + // Bind and immediately close to get a port that's definitely not in use + const server = createServer(); + await new Promise((resolve) => server.listen(0, resolve)); + const port = (server.address() as { port: number }).port; + await new Promise((resolve) => server.close(() => resolve())); + + process.env.AIKIDO_REALTIME_ENDPOINT = `http://localhost:${port}/`; + process.env.AIKIDO_DEBUG_SSE = "true"; + + const logger = new LoggerForTesting(); + + connectToSSE({ + token: new Token("test-token"), + logger, + onEvent() {}, + }); + + await new Promise((r) => setTimeout(r, 500)); + + t.ok(logger.getMessages().some((m) => m.includes("SSE connection error:"))); +}); diff --git a/library/agent/realtime/connectToSSE.events.test.ts b/library/agent/realtime/connectToSSE.events.test.ts new file mode 100644 index 000000000..3cc5b0c35 --- /dev/null +++ b/library/agent/realtime/connectToSSE.events.test.ts @@ -0,0 +1,55 @@ +import * as t from "tap"; +import { createServer, type ServerResponse } from "http"; +import { Token } from "../api/Token"; +import { LoggerForTesting } from "../logger/LoggerForTesting"; +import { connectToSSE } from "./connectToSSE"; +import type { EventSourceMessage } from "../../helpers/eventsource-parser/types"; + +t.test("it connects and receives config-updated events", async (t) => { + let sseRes: ServerResponse | null = null; + + const server = createServer((_req, res) => { + sseRes = res; + res.writeHead(200, { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }); + + const data = JSON.stringify({ serviceId: 1, configUpdatedAt: 100 }); + res.write(`event: config-updated\ndata: ${data}\n\n`); + }); + + await new Promise((resolve) => server.listen(0, resolve)); + const port = (server.address() as { port: number }).port; + process.env.AIKIDO_REALTIME_ENDPOINT = `http://localhost:${port}/`; + + const events: EventSourceMessage[] = []; + + try { + connectToSSE({ + token: new Token("test-token"), + logger: new LoggerForTesting(), + onEvent(event) { + events.push(event); + }, + }); + + await new Promise((r) => setTimeout(r, 200)); + + t.equal(events.length, 1); + t.equal(events[0].event, "config-updated"); + t.same(JSON.parse(events[0].data), { serviceId: 1, configUpdatedAt: 100 }); + + const data2 = JSON.stringify({ serviceId: 1, configUpdatedAt: 200 }); + sseRes!.write(`event: config-updated\ndata: ${data2}\n\n`); + + await new Promise((r) => setTimeout(r, 100)); + + t.equal(events.length, 2); + t.same(JSON.parse(events[1].data), { serviceId: 1, configUpdatedAt: 200 }); + } finally { + server.closeAllConnections(); + server.close(); + } +}); diff --git a/library/agent/realtime/connectToSSE.ping.test.ts b/library/agent/realtime/connectToSSE.ping.test.ts new file mode 100644 index 000000000..f744104e7 --- /dev/null +++ b/library/agent/realtime/connectToSSE.ping.test.ts @@ -0,0 +1,41 @@ +import * as t from "tap"; +import { createServer } from "http"; +import { Token } from "../api/Token"; +import { LoggerForTesting } from "../logger/LoggerForTesting"; +import { connectToSSE } from "./connectToSSE"; +import type { EventSourceMessage } from "../../helpers/eventsource-parser/types"; + +t.test("it receives pings without emitting events", async (t) => { + const server = createServer((_req, res) => { + res.writeHead(200, { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }); + + res.write(": ping\n\n"); + }); + + await new Promise((resolve) => server.listen(0, resolve)); + const port = (server.address() as { port: number }).port; + process.env.AIKIDO_REALTIME_ENDPOINT = `http://localhost:${port}/`; + + const events: EventSourceMessage[] = []; + + try { + connectToSSE({ + token: new Token("test-token"), + logger: new LoggerForTesting(), + onEvent(event) { + events.push(event); + }, + }); + + await new Promise((r) => setTimeout(r, 200)); + + t.equal(events.length, 0); + } finally { + server.closeAllConnections(); + server.close(); + } +}); diff --git a/library/agent/realtime/connectToSSE.reconnect.test.ts b/library/agent/realtime/connectToSSE.reconnect.test.ts new file mode 100644 index 000000000..5f3a7e014 --- /dev/null +++ b/library/agent/realtime/connectToSSE.reconnect.test.ts @@ -0,0 +1,46 @@ +import * as t from "tap"; +import { createServer, type ServerResponse } from "http"; +import { Token } from "../api/Token"; +import { LoggerForTesting } from "../logger/LoggerForTesting"; +import { connectToSSE } from "./connectToSSE"; + +t.test("it reconnects when server closes connection", async (t) => { + let connectionCount = 0; + let sseRes: ServerResponse | null = null; + + const server = createServer((_req, res) => { + connectionCount++; + sseRes = res; + res.writeHead(200, { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }); + res.write(": ping\n\n"); + }); + + await new Promise((resolve) => server.listen(0, resolve)); + const port = (server.address() as { port: number }).port; + process.env.AIKIDO_REALTIME_ENDPOINT = `http://localhost:${port}/`; + + try { + connectToSSE({ + token: new Token("test-token"), + logger: new LoggerForTesting(), + onEvent() {}, + }); + + await new Promise((r) => setTimeout(r, 200)); + t.equal(connectionCount, 1); + + sseRes!.end(); + + // Wait for reconnect (initial delay is 5s + up to 2.5s jitter) + await new Promise((r) => setTimeout(r, 8000)); + + t.equal(connectionCount, 2); + } finally { + server.closeAllConnections(); + server.close(); + } +}); From 1d0592295ad631c98fb8425a97134fecb65485ec Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Thu, 21 May 2026 10:54:07 +0200 Subject: [PATCH 40/47] Consolidate SSE unit tests into fewer files --- .../agent/realtime/connectToSSE.auth.test.ts | 38 ---------- .../realtime/connectToSSE.events.test.ts | 55 --------------- .../agent/realtime/connectToSSE.ping.test.ts | 41 ----------- library/agent/realtime/connectToSSE.test.ts | 69 +++++++++++++++++++ 4 files changed, 69 insertions(+), 134 deletions(-) delete mode 100644 library/agent/realtime/connectToSSE.auth.test.ts delete mode 100644 library/agent/realtime/connectToSSE.events.test.ts delete mode 100644 library/agent/realtime/connectToSSE.ping.test.ts create mode 100644 library/agent/realtime/connectToSSE.test.ts diff --git a/library/agent/realtime/connectToSSE.auth.test.ts b/library/agent/realtime/connectToSSE.auth.test.ts deleted file mode 100644 index 9753e6923..000000000 --- a/library/agent/realtime/connectToSSE.auth.test.ts +++ /dev/null @@ -1,38 +0,0 @@ -import * as t from "tap"; -import { createServer } from "http"; -import { Token } from "../api/Token"; -import { LoggerForTesting } from "../logger/LoggerForTesting"; -import { connectToSSE } from "./connectToSSE"; - -t.test("it sends Authorization header", async (t) => { - let receivedAuth: string | undefined; - - const server = createServer((req, res) => { - receivedAuth = req.headers.authorization; - res.writeHead(200, { - "Content-Type": "text/event-stream", - "Cache-Control": "no-cache", - Connection: "keep-alive", - }); - res.write(": ping\n\n"); - }); - - await new Promise((resolve) => server.listen(0, resolve)); - const port = (server.address() as { port: number }).port; - process.env.AIKIDO_REALTIME_ENDPOINT = `http://localhost:${port}/`; - - try { - connectToSSE({ - token: new Token("my-secret-token"), - logger: new LoggerForTesting(), - onEvent() {}, - }); - - await new Promise((r) => setTimeout(r, 200)); - - t.equal(receivedAuth, "my-secret-token"); - } finally { - server.closeAllConnections(); - server.close(); - } -}); diff --git a/library/agent/realtime/connectToSSE.events.test.ts b/library/agent/realtime/connectToSSE.events.test.ts deleted file mode 100644 index 3cc5b0c35..000000000 --- a/library/agent/realtime/connectToSSE.events.test.ts +++ /dev/null @@ -1,55 +0,0 @@ -import * as t from "tap"; -import { createServer, type ServerResponse } from "http"; -import { Token } from "../api/Token"; -import { LoggerForTesting } from "../logger/LoggerForTesting"; -import { connectToSSE } from "./connectToSSE"; -import type { EventSourceMessage } from "../../helpers/eventsource-parser/types"; - -t.test("it connects and receives config-updated events", async (t) => { - let sseRes: ServerResponse | null = null; - - const server = createServer((_req, res) => { - sseRes = res; - res.writeHead(200, { - "Content-Type": "text/event-stream", - "Cache-Control": "no-cache", - Connection: "keep-alive", - }); - - const data = JSON.stringify({ serviceId: 1, configUpdatedAt: 100 }); - res.write(`event: config-updated\ndata: ${data}\n\n`); - }); - - await new Promise((resolve) => server.listen(0, resolve)); - const port = (server.address() as { port: number }).port; - process.env.AIKIDO_REALTIME_ENDPOINT = `http://localhost:${port}/`; - - const events: EventSourceMessage[] = []; - - try { - connectToSSE({ - token: new Token("test-token"), - logger: new LoggerForTesting(), - onEvent(event) { - events.push(event); - }, - }); - - await new Promise((r) => setTimeout(r, 200)); - - t.equal(events.length, 1); - t.equal(events[0].event, "config-updated"); - t.same(JSON.parse(events[0].data), { serviceId: 1, configUpdatedAt: 100 }); - - const data2 = JSON.stringify({ serviceId: 1, configUpdatedAt: 200 }); - sseRes!.write(`event: config-updated\ndata: ${data2}\n\n`); - - await new Promise((r) => setTimeout(r, 100)); - - t.equal(events.length, 2); - t.same(JSON.parse(events[1].data), { serviceId: 1, configUpdatedAt: 200 }); - } finally { - server.closeAllConnections(); - server.close(); - } -}); diff --git a/library/agent/realtime/connectToSSE.ping.test.ts b/library/agent/realtime/connectToSSE.ping.test.ts deleted file mode 100644 index f744104e7..000000000 --- a/library/agent/realtime/connectToSSE.ping.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import * as t from "tap"; -import { createServer } from "http"; -import { Token } from "../api/Token"; -import { LoggerForTesting } from "../logger/LoggerForTesting"; -import { connectToSSE } from "./connectToSSE"; -import type { EventSourceMessage } from "../../helpers/eventsource-parser/types"; - -t.test("it receives pings without emitting events", async (t) => { - const server = createServer((_req, res) => { - res.writeHead(200, { - "Content-Type": "text/event-stream", - "Cache-Control": "no-cache", - Connection: "keep-alive", - }); - - res.write(": ping\n\n"); - }); - - await new Promise((resolve) => server.listen(0, resolve)); - const port = (server.address() as { port: number }).port; - process.env.AIKIDO_REALTIME_ENDPOINT = `http://localhost:${port}/`; - - const events: EventSourceMessage[] = []; - - try { - connectToSSE({ - token: new Token("test-token"), - logger: new LoggerForTesting(), - onEvent(event) { - events.push(event); - }, - }); - - await new Promise((r) => setTimeout(r, 200)); - - t.equal(events.length, 0); - } finally { - server.closeAllConnections(); - server.close(); - } -}); diff --git a/library/agent/realtime/connectToSSE.test.ts b/library/agent/realtime/connectToSSE.test.ts new file mode 100644 index 000000000..0fb485e89 --- /dev/null +++ b/library/agent/realtime/connectToSSE.test.ts @@ -0,0 +1,69 @@ +import * as t from "tap"; +import { createServer, type ServerResponse } from "http"; +import { Token } from "../api/Token"; +import { LoggerForTesting } from "../logger/LoggerForTesting"; +import { connectToSSE } from "./connectToSSE"; +import type { EventSourceMessage } from "../../helpers/eventsource-parser/types"; + +t.test( + "it connects with auth header and receives events, ignoring pings", + async (t) => { + let receivedAuth: string | undefined; + let sseRes: ServerResponse | null = null; + + const server = createServer((req, res) => { + receivedAuth = req.headers.authorization; + sseRes = res; + res.writeHead(200, { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }); + + res.write(": ping\n\n"); + + const data = JSON.stringify({ serviceId: 1, configUpdatedAt: 100 }); + res.write(`event: config-updated\ndata: ${data}\n\n`); + }); + + await new Promise((resolve) => server.listen(0, resolve)); + const port = (server.address() as { port: number }).port; + process.env.AIKIDO_REALTIME_ENDPOINT = `http://localhost:${port}/`; + + const events: EventSourceMessage[] = []; + + try { + connectToSSE({ + token: new Token("my-secret-token"), + logger: new LoggerForTesting(), + onEvent(event) { + events.push(event); + }, + }); + + await new Promise((r) => setTimeout(r, 200)); + + t.equal(receivedAuth, "my-secret-token"); + t.equal(events.length, 1); + t.equal(events[0].event, "config-updated"); + t.same(JSON.parse(events[0].data), { + serviceId: 1, + configUpdatedAt: 100, + }); + + const data2 = JSON.stringify({ serviceId: 1, configUpdatedAt: 200 }); + sseRes!.write(`event: config-updated\ndata: ${data2}\n\n`); + + await new Promise((r) => setTimeout(r, 100)); + + t.equal(events.length, 2); + t.same(JSON.parse(events[1].data), { + serviceId: 1, + configUpdatedAt: 200, + }); + } finally { + server.closeAllConnections(); + server.close(); + } + } +); From ea5c50c2e052a038f476b9ecc09d51a01aebd4d2 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Thu, 21 May 2026 11:40:29 +0200 Subject: [PATCH 41/47] Add unit test for SSE read timeout --- .../realtime/connectToSSE.timeout.test.ts | 41 +++++++++++++++++++ library/agent/realtime/connectToSSE.ts | 8 +++- 2 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 library/agent/realtime/connectToSSE.timeout.test.ts diff --git a/library/agent/realtime/connectToSSE.timeout.test.ts b/library/agent/realtime/connectToSSE.timeout.test.ts new file mode 100644 index 000000000..0e36c6308 --- /dev/null +++ b/library/agent/realtime/connectToSSE.timeout.test.ts @@ -0,0 +1,41 @@ +import * as t from "tap"; +import { createServer } from "http"; +import { Token } from "../api/Token"; +import { LoggerForTesting } from "../logger/LoggerForTesting"; +import { connectToSSE } from "./connectToSSE"; + +t.test("it reconnects on read timeout", async (t) => { + let connectionCount = 0; + + const server = createServer((_req, res) => { + connectionCount++; + res.writeHead(200, { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }); + }); + + await new Promise((resolve) => server.listen(0, resolve)); + const port = (server.address() as { port: number }).port; + process.env.AIKIDO_REALTIME_ENDPOINT = `http://localhost:${port}/`; + + try { + connectToSSE({ + token: new Token("test-token"), + logger: new LoggerForTesting(), + onEvent() {}, + readTimeoutMs: 200, + initialReconnectMs: 100, + }); + + await new Promise((r) => setTimeout(r, 200)); + t.equal(connectionCount, 1); + + await new Promise((r) => setTimeout(r, 500)); + t.equal(connectionCount, 2); + } finally { + server.closeAllConnections(); + server.close(); + } +}); diff --git a/library/agent/realtime/connectToSSE.ts b/library/agent/realtime/connectToSSE.ts index f39c19e5b..7ff96df56 100644 --- a/library/agent/realtime/connectToSSE.ts +++ b/library/agent/realtime/connectToSSE.ts @@ -16,12 +16,16 @@ export function connectToSSE({ token, logger, onEvent, + initialReconnectMs = INITIAL_RECONNECT_MS, + readTimeoutMs = READ_TIMEOUT_MS, }: { token: Token; logger: Logger; onEvent: (event: EventSourceMessage) => void; + initialReconnectMs?: number; + readTimeoutMs?: number; }) { - let reconnectMs = INITIAL_RECONNECT_MS; + let reconnectMs = initialReconnectMs; let reconnectTimer: NodeJS.Timeout | null = null; let currentRequest: ReturnType | null = null; @@ -108,7 +112,7 @@ export function connectToSSE({ currentRequest = req; req.on("socket", (socket) => { - socket.setTimeout(READ_TIMEOUT_MS, () => { + socket.setTimeout(readTimeoutMs, () => { if (socket.destroyed) { return; } From 29f937f4314b7b9b4c5ae178c8ef7520010ef241 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Thu, 21 May 2026 11:53:13 +0200 Subject: [PATCH 42/47] Reset backoff on SSE socket timeout --- library/agent/realtime/connectToSSE.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/library/agent/realtime/connectToSSE.ts b/library/agent/realtime/connectToSSE.ts index 7ff96df56..2f6683b71 100644 --- a/library/agent/realtime/connectToSSE.ts +++ b/library/agent/realtime/connectToSSE.ts @@ -45,6 +45,8 @@ export function connectToSSE({ currentRequest = null; } + let connectedAt: number | undefined; + const url = new URL(`${getRealtimeURL().toString()}api/runtime/stream`); logDebug(`SSE connecting to ${url.toString()}`); @@ -77,7 +79,7 @@ export function connectToSSE({ return; } - const connectedAt = Date.now(); + connectedAt = Date.now(); logDebug("SSE connected successfully"); const parser = createParser({ @@ -117,6 +119,9 @@ export function connectToSSE({ return; } logDebug("SSE read timeout, reconnecting"); + if (connectedAt) { + resetBackoffIfStable(connectedAt); + } req.destroy(); }); // Don't keep the process alive just for the SSE connection From e3656fa7c9e8159b5d6a4ab61eea4be841b1d35b Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Thu, 21 May 2026 12:19:40 +0200 Subject: [PATCH 43/47] Split connectToSSE into connect + reconnect loop The single function mixed HTTP/SSE plumbing with reconnect policy (backoff, jitter, dedup guards). Now connect() returns a promise that resolves with { outcome, statusCode } when the connection ends, and the reconnect loop decides what to do with the result. --- .../realtime-config-updates.test.mjs | 2 +- library/agent/realtime/connectToSSE.ts | 154 +++++++++--------- 2 files changed, 82 insertions(+), 74 deletions(-) diff --git a/end2end/tests-new/realtime-config-updates.test.mjs b/end2end/tests-new/realtime-config-updates.test.mjs index 54bf0a462..8bb8e7b9a 100644 --- a/end2end/tests-new/realtime-config-updates.test.mjs +++ b/end2end/tests-new/realtime-config-updates.test.mjs @@ -137,7 +137,7 @@ test("it reconnects SSE after server disconnects", async () => { // Wait for reconnect (initial reconnect delay is 5s + jitter up to 7.5s) await timeout(10000); - match(stdout, /SSE connection closed by server, reconnecting/); + match(stdout, /SSE connection closed by server/); // Verify SSE reconnected const connectedCount = diff --git a/library/agent/realtime/connectToSSE.ts b/library/agent/realtime/connectToSSE.ts index 2f6683b71..bcf2d06e6 100644 --- a/library/agent/realtime/connectToSSE.ts +++ b/library/agent/realtime/connectToSSE.ts @@ -12,41 +12,39 @@ const MAX_RECONNECT_MS = 60 * 1000; const STABLE_CONNECTION_MS = 30 * 1000; const READ_TIMEOUT_MS = 70 * 1000; -export function connectToSSE({ +type ConnectResult = + | { outcome: "error" } + | { outcome: "disconnected"; statusCode: number }; + +function delay(ms: number): Promise { + return new Promise((resolve) => { + const timer = setTimeout(resolve, ms); + timer.unref(); + }); +} + +function connect({ token, - logger, onEvent, - initialReconnectMs = INITIAL_RECONNECT_MS, - readTimeoutMs = READ_TIMEOUT_MS, + readTimeoutMs, + logDebug, }: { token: Token; - logger: Logger; onEvent: (event: EventSourceMessage) => void; - initialReconnectMs?: number; - readTimeoutMs?: number; -}) { - let reconnectMs = initialReconnectMs; - let reconnectTimer: NodeJS.Timeout | null = null; - let currentRequest: ReturnType | null = null; - - const debugSSE = isDebuggingSSE(); - - function logDebug(msg: string) { - if (debugSSE) { - logger.log(msg); - } - } - - function connect() { - reconnectScheduled = false; - - if (currentRequest) { - currentRequest.destroy(); - currentRequest = null; + readTimeoutMs: number; + logDebug: (msg: string) => void; +}): Promise { + return new Promise((resolve) => { + let resolved = false; + + function resolveOnce(result: ConnectResult) { + if (resolved) { + return; + } + resolved = true; + resolve(result); } - let connectedAt: number | undefined; - const url = new URL(`${getRealtimeURL().toString()}api/runtime/stream`); logDebug(`SSE connecting to ${url.toString()}`); @@ -64,22 +62,14 @@ export function connectToSSE({ }, }, (response) => { - if (response.statusCode === 401 || response.statusCode === 403) { - logger.log( - `SSE connection rejected with status ${response.statusCode}, stopping` - ); - response.destroy(); - return; - } + const statusCode = response.statusCode!; - if (response.statusCode !== 200) { - logDebug(`SSE connection failed with status ${response.statusCode}`); + if (statusCode !== 200) { response.destroy(); - scheduleReconnect(); + resolveOnce({ outcome: "disconnected", statusCode }); return; } - connectedAt = Date.now(); logDebug("SSE connected successfully"); const parser = createParser({ @@ -96,74 +86,92 @@ export function connectToSSE({ }); response.on("end", () => { - logDebug("SSE connection closed by server, reconnecting"); + logDebug("SSE connection closed by server"); parser.reset(); - resetBackoffIfStable(connectedAt); - scheduleReconnect(); + resolveOnce({ outcome: "disconnected", statusCode }); }); response.on("error", (error) => { logDebug(`SSE stream error: ${error.message}`); parser.reset(); - resetBackoffIfStable(connectedAt); - scheduleReconnect(); + resolveOnce({ outcome: "disconnected", statusCode }); }); } ); - currentRequest = req; - req.on("socket", (socket) => { socket.setTimeout(readTimeoutMs, () => { if (socket.destroyed) { return; } - logDebug("SSE read timeout, reconnecting"); - if (connectedAt) { - resetBackoffIfStable(connectedAt); - } + logDebug("SSE read timeout"); + resolveOnce({ outcome: "error" }); req.destroy(); }); - // Don't keep the process alive just for the SSE connection socket.unref(); }); req.on("error", (error) => { logDebug(`SSE connection error: ${error.message}`); - scheduleReconnect(); + resolveOnce({ outcome: "error" }); }); req.end(); - } + }); +} - function resetBackoffIfStable(connectedAt: number) { - if (Date.now() - connectedAt >= STABLE_CONNECTION_MS) { - reconnectMs = INITIAL_RECONNECT_MS; +export function connectToSSE({ + token, + logger, + onEvent, + initialReconnectMs = INITIAL_RECONNECT_MS, + readTimeoutMs = READ_TIMEOUT_MS, +}: { + token: Token; + logger: Logger; + onEvent: (event: EventSourceMessage) => void; + initialReconnectMs?: number; + readTimeoutMs?: number; +}) { + let reconnectMs = initialReconnectMs; + + const debugSSE = isDebuggingSSE(); + + function logDebug(msg: string) { + if (debugSSE) { + logger.log(msg); } } - let reconnectScheduled = false; + async function loop() { + while (true) { + const start = Date.now(); + const result = await connect({ token, onEvent, readTimeoutMs, logDebug }); + + if ( + result.outcome === "disconnected" && + (result.statusCode === 401 || result.statusCode === 403) + ) { + logger.log( + `SSE connection rejected with status ${result.statusCode}, stopping` + ); + return; + } - function scheduleReconnect() { - if (reconnectScheduled) { - return; - } - reconnectScheduled = true; + if (Date.now() - start >= STABLE_CONNECTION_MS) { + reconnectMs = initialReconnectMs; + } - if (reconnectTimer) { - clearTimeout(reconnectTimer); - } + const jitter = Math.random() * (reconnectMs / 2); + const delayMs = reconnectMs + jitter; - const jitter = Math.random() * (reconnectMs / 2); - const delay = reconnectMs + jitter; + logDebug(`SSE scheduling reconnect in ${Math.round(delayMs)}ms`); - logDebug(`SSE scheduling reconnect in ${Math.round(delay)}ms`); + reconnectMs = Math.min(reconnectMs * 2, MAX_RECONNECT_MS); - reconnectTimer = setTimeout(connect, delay); - reconnectMs = Math.min(reconnectMs * 2, MAX_RECONNECT_MS); - // Don't keep the process alive just for the reconnect timer - reconnectTimer.unref(); + await delay(delayMs); + } } - connect(); + loop().catch(() => {}); } From 18fd83254f19f5cdd957e622b32e204901b172dd Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Thu, 21 May 2026 12:21:48 +0200 Subject: [PATCH 44/47] Log SSE loop errors instead of swallowing them --- library/agent/realtime/connectToSSE.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/library/agent/realtime/connectToSSE.ts b/library/agent/realtime/connectToSSE.ts index bcf2d06e6..c277c111b 100644 --- a/library/agent/realtime/connectToSSE.ts +++ b/library/agent/realtime/connectToSSE.ts @@ -173,5 +173,7 @@ export function connectToSSE({ } } - loop().catch(() => {}); + loop().catch((error) => { + logger.log(`SSE loop error: ${error.message}`); + }); } From 768dfa86477324a6694e7d3da151f7484b14d357 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Thu, 21 May 2026 17:26:12 +0200 Subject: [PATCH 45/47] Fix SSE test compat with Node 16 and Node 26 closeAllConnections() doesn't exist on Node 16, causing the finally block to throw before server.close() runs, hanging the test. Replace with server.unref() and socket.unref() so the process exits cleanly. The ESM test helper passed undefined as the message arg to assert.partialDeepStrictEqual, which Node 26 rejects. Use rest params to avoid passing it when omitted. --- library/agent/realtime/connectToSSE.401.test.ts | 3 ++- library/agent/realtime/connectToSSE.500.test.ts | 3 ++- library/agent/realtime/connectToSSE.reconnect.test.ts | 3 ++- library/agent/realtime/connectToSSE.test.ts | 3 ++- library/agent/realtime/connectToSSE.timeout.test.ts | 3 ++- scripts/helpers/test-helpers.mjs | 6 +++--- 6 files changed, 13 insertions(+), 8 deletions(-) diff --git a/library/agent/realtime/connectToSSE.401.test.ts b/library/agent/realtime/connectToSSE.401.test.ts index be6474cf0..3796e1e89 100644 --- a/library/agent/realtime/connectToSSE.401.test.ts +++ b/library/agent/realtime/connectToSSE.401.test.ts @@ -14,6 +14,8 @@ t.test("it stops reconnecting on 401", async (t) => { }); await new Promise((resolve) => server.listen(0, resolve)); + server.unref(); + server.on("connection", (socket) => socket.unref()); const port = (server.address() as { port: number }).port; process.env.AIKIDO_REALTIME_ENDPOINT = `http://localhost:${port}/`; @@ -33,7 +35,6 @@ t.test("it stops reconnecting on 401", async (t) => { /SSE connection rejected with status 401, stopping/, ]); } finally { - server.closeAllConnections(); server.close(); } }); diff --git a/library/agent/realtime/connectToSSE.500.test.ts b/library/agent/realtime/connectToSSE.500.test.ts index effae28e3..237892d93 100644 --- a/library/agent/realtime/connectToSSE.500.test.ts +++ b/library/agent/realtime/connectToSSE.500.test.ts @@ -23,6 +23,8 @@ t.test("it reconnects on non-200 status", async (t) => { }); await new Promise((resolve) => server.listen(0, resolve)); + server.unref(); + server.on("connection", (socket) => socket.unref()); const port = (server.address() as { port: number }).port; process.env.AIKIDO_REALTIME_ENDPOINT = `http://localhost:${port}/`; @@ -38,7 +40,6 @@ t.test("it reconnects on non-200 status", async (t) => { t.equal(connectionCount, 2); } finally { - server.closeAllConnections(); server.close(); } }); diff --git a/library/agent/realtime/connectToSSE.reconnect.test.ts b/library/agent/realtime/connectToSSE.reconnect.test.ts index 5f3a7e014..c67e782a7 100644 --- a/library/agent/realtime/connectToSSE.reconnect.test.ts +++ b/library/agent/realtime/connectToSSE.reconnect.test.ts @@ -20,6 +20,8 @@ t.test("it reconnects when server closes connection", async (t) => { }); await new Promise((resolve) => server.listen(0, resolve)); + server.unref(); + server.on("connection", (socket) => socket.unref()); const port = (server.address() as { port: number }).port; process.env.AIKIDO_REALTIME_ENDPOINT = `http://localhost:${port}/`; @@ -40,7 +42,6 @@ t.test("it reconnects when server closes connection", async (t) => { t.equal(connectionCount, 2); } finally { - server.closeAllConnections(); server.close(); } }); diff --git a/library/agent/realtime/connectToSSE.test.ts b/library/agent/realtime/connectToSSE.test.ts index 0fb485e89..90aaaeba6 100644 --- a/library/agent/realtime/connectToSSE.test.ts +++ b/library/agent/realtime/connectToSSE.test.ts @@ -27,6 +27,8 @@ t.test( }); await new Promise((resolve) => server.listen(0, resolve)); + server.unref(); + server.on("connection", (socket) => socket.unref()); const port = (server.address() as { port: number }).port; process.env.AIKIDO_REALTIME_ENDPOINT = `http://localhost:${port}/`; @@ -62,7 +64,6 @@ t.test( configUpdatedAt: 200, }); } finally { - server.closeAllConnections(); server.close(); } } diff --git a/library/agent/realtime/connectToSSE.timeout.test.ts b/library/agent/realtime/connectToSSE.timeout.test.ts index 0e36c6308..b0bbec974 100644 --- a/library/agent/realtime/connectToSSE.timeout.test.ts +++ b/library/agent/realtime/connectToSSE.timeout.test.ts @@ -17,6 +17,8 @@ t.test("it reconnects on read timeout", async (t) => { }); await new Promise((resolve) => server.listen(0, resolve)); + server.unref(); + server.on("connection", (socket) => socket.unref()); const port = (server.address() as { port: number }).port; process.env.AIKIDO_REALTIME_ENDPOINT = `http://localhost:${port}/`; @@ -35,7 +37,6 @@ t.test("it reconnects on read timeout", async (t) => { await new Promise((r) => setTimeout(r, 500)); t.equal(connectionCount, 2); } finally { - server.closeAllConnections(); server.close(); } }); diff --git a/scripts/helpers/test-helpers.mjs b/scripts/helpers/test-helpers.mjs index e74df83ef..5e881b3f8 100644 --- a/scripts/helpers/test-helpers.mjs +++ b/scripts/helpers/test-helpers.mjs @@ -31,7 +31,7 @@ export function throws(...args) { assert.fail("Missing expected exception"); } -export function match(actual, expected, message) { +export function match(actual, expected, ...rest) { if (typeof expected === "string") { expected = new RegExp(RegExp.escape(expected)); } @@ -41,11 +41,11 @@ export function match(actual, expected, message) { actual = String(actual); } - assert.match(actual, expected, message); + assert.match(actual, expected, ...rest); return; } - assert.partialDeepStrictEqual(actual, expected, message); + assert.partialDeepStrictEqual(actual, expected, ...rest); } function toPlainObject(value) { From 8520477e124bd5b4e0a3ac7015d58d19b98378f9 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Thu, 21 May 2026 17:44:03 +0200 Subject: [PATCH 46/47] Avoid t.match with regex inside array in 401 test --- library/agent/realtime/connectToSSE.401.test.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/library/agent/realtime/connectToSSE.401.test.ts b/library/agent/realtime/connectToSSE.401.test.ts index 3796e1e89..5e042bce9 100644 --- a/library/agent/realtime/connectToSSE.401.test.ts +++ b/library/agent/realtime/connectToSSE.401.test.ts @@ -31,9 +31,11 @@ t.test("it stops reconnecting on 401", async (t) => { await new Promise((r) => setTimeout(r, 500)); t.equal(connectionCount, 1); - t.match(logger.getMessages(), [ - /SSE connection rejected with status 401, stopping/, - ]); + t.equal(logger.getMessages().length, 1); + t.match( + logger.getMessages()[0], + /SSE connection rejected with status 401, stopping/ + ); } finally { server.close(); } From 16a288a5880380ad540017397d56e3597b6d4a71 Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Fri, 22 May 2026 12:21:48 +0200 Subject: [PATCH 47/47] Use setTimeout from node:timers/promises --- library/agent/realtime/connectToSSE.401.test.ts | 3 ++- library/agent/realtime/connectToSSE.500.test.ts | 3 ++- .../agent/realtime/connectToSSE.connrefused.test.ts | 3 ++- library/agent/realtime/connectToSSE.reconnect.test.ts | 5 +++-- library/agent/realtime/connectToSSE.test.ts | 5 +++-- library/agent/realtime/connectToSSE.timeout.test.ts | 5 +++-- library/agent/realtime/connectToSSE.ts | 11 +++-------- 7 files changed, 18 insertions(+), 17 deletions(-) diff --git a/library/agent/realtime/connectToSSE.401.test.ts b/library/agent/realtime/connectToSSE.401.test.ts index 5e042bce9..a3a258aae 100644 --- a/library/agent/realtime/connectToSSE.401.test.ts +++ b/library/agent/realtime/connectToSSE.401.test.ts @@ -1,4 +1,5 @@ import * as t from "tap"; +import { setTimeout } from "node:timers/promises"; import { createServer } from "http"; import { Token } from "../api/Token"; import { LoggerForTesting } from "../logger/LoggerForTesting"; @@ -28,7 +29,7 @@ t.test("it stops reconnecting on 401", async (t) => { onEvent() {}, }); - await new Promise((r) => setTimeout(r, 500)); + await setTimeout(500); t.equal(connectionCount, 1); t.equal(logger.getMessages().length, 1); diff --git a/library/agent/realtime/connectToSSE.500.test.ts b/library/agent/realtime/connectToSSE.500.test.ts index 237892d93..667002208 100644 --- a/library/agent/realtime/connectToSSE.500.test.ts +++ b/library/agent/realtime/connectToSSE.500.test.ts @@ -1,4 +1,5 @@ import * as t from "tap"; +import { setTimeout } from "node:timers/promises"; import { createServer } from "http"; import { Token } from "../api/Token"; import { LoggerForTesting } from "../logger/LoggerForTesting"; @@ -36,7 +37,7 @@ t.test("it reconnects on non-200 status", async (t) => { }); // Wait for reconnect after 500 (initial delay 5s + up to 2.5s jitter) - await new Promise((r) => setTimeout(r, 8000)); + await setTimeout(8000); t.equal(connectionCount, 2); } finally { diff --git a/library/agent/realtime/connectToSSE.connrefused.test.ts b/library/agent/realtime/connectToSSE.connrefused.test.ts index 48a79e78a..6c2ef323b 100644 --- a/library/agent/realtime/connectToSSE.connrefused.test.ts +++ b/library/agent/realtime/connectToSSE.connrefused.test.ts @@ -1,4 +1,5 @@ import * as t from "tap"; +import { setTimeout } from "node:timers/promises"; import { createServer } from "http"; import { Token } from "../api/Token"; import { LoggerForTesting } from "../logger/LoggerForTesting"; @@ -22,7 +23,7 @@ t.test("it handles connection refused", async (t) => { onEvent() {}, }); - await new Promise((r) => setTimeout(r, 500)); + await setTimeout(500); t.ok(logger.getMessages().some((m) => m.includes("SSE connection error:"))); }); diff --git a/library/agent/realtime/connectToSSE.reconnect.test.ts b/library/agent/realtime/connectToSSE.reconnect.test.ts index c67e782a7..a5fc4f669 100644 --- a/library/agent/realtime/connectToSSE.reconnect.test.ts +++ b/library/agent/realtime/connectToSSE.reconnect.test.ts @@ -1,4 +1,5 @@ import * as t from "tap"; +import { setTimeout } from "node:timers/promises"; import { createServer, type ServerResponse } from "http"; import { Token } from "../api/Token"; import { LoggerForTesting } from "../logger/LoggerForTesting"; @@ -32,13 +33,13 @@ t.test("it reconnects when server closes connection", async (t) => { onEvent() {}, }); - await new Promise((r) => setTimeout(r, 200)); + await setTimeout(200); t.equal(connectionCount, 1); sseRes!.end(); // Wait for reconnect (initial delay is 5s + up to 2.5s jitter) - await new Promise((r) => setTimeout(r, 8000)); + await setTimeout(8000); t.equal(connectionCount, 2); } finally { diff --git a/library/agent/realtime/connectToSSE.test.ts b/library/agent/realtime/connectToSSE.test.ts index 90aaaeba6..699d948ba 100644 --- a/library/agent/realtime/connectToSSE.test.ts +++ b/library/agent/realtime/connectToSSE.test.ts @@ -1,4 +1,5 @@ import * as t from "tap"; +import { setTimeout } from "node:timers/promises"; import { createServer, type ServerResponse } from "http"; import { Token } from "../api/Token"; import { LoggerForTesting } from "../logger/LoggerForTesting"; @@ -43,7 +44,7 @@ t.test( }, }); - await new Promise((r) => setTimeout(r, 200)); + await setTimeout(200); t.equal(receivedAuth, "my-secret-token"); t.equal(events.length, 1); @@ -56,7 +57,7 @@ t.test( const data2 = JSON.stringify({ serviceId: 1, configUpdatedAt: 200 }); sseRes!.write(`event: config-updated\ndata: ${data2}\n\n`); - await new Promise((r) => setTimeout(r, 100)); + await setTimeout(100); t.equal(events.length, 2); t.same(JSON.parse(events[1].data), { diff --git a/library/agent/realtime/connectToSSE.timeout.test.ts b/library/agent/realtime/connectToSSE.timeout.test.ts index b0bbec974..b378f88b6 100644 --- a/library/agent/realtime/connectToSSE.timeout.test.ts +++ b/library/agent/realtime/connectToSSE.timeout.test.ts @@ -1,4 +1,5 @@ import * as t from "tap"; +import { setTimeout } from "node:timers/promises"; import { createServer } from "http"; import { Token } from "../api/Token"; import { LoggerForTesting } from "../logger/LoggerForTesting"; @@ -31,10 +32,10 @@ t.test("it reconnects on read timeout", async (t) => { initialReconnectMs: 100, }); - await new Promise((r) => setTimeout(r, 200)); + await setTimeout(200); t.equal(connectionCount, 1); - await new Promise((r) => setTimeout(r, 500)); + await setTimeout(500); t.equal(connectionCount, 2); } finally { server.close(); diff --git a/library/agent/realtime/connectToSSE.ts b/library/agent/realtime/connectToSSE.ts index c277c111b..ff6bbed4f 100644 --- a/library/agent/realtime/connectToSSE.ts +++ b/library/agent/realtime/connectToSSE.ts @@ -1,5 +1,6 @@ import { request as requestHttp } from "http"; import { request as requestHttps } from "https"; +import { setTimeout } from "node:timers/promises"; import { createParser } from "../../helpers/eventsource-parser/parse"; import type { EventSourceMessage } from "../../helpers/eventsource-parser/types"; import { isDebuggingSSE } from "../../helpers/isDebuggingSSE"; @@ -16,13 +17,6 @@ type ConnectResult = | { outcome: "error" } | { outcome: "disconnected"; statusCode: number }; -function delay(ms: number): Promise { - return new Promise((resolve) => { - const timer = setTimeout(resolve, ms); - timer.unref(); - }); -} - function connect({ token, onEvent, @@ -169,7 +163,8 @@ export function connectToSSE({ reconnectMs = Math.min(reconnectMs * 2, MAX_RECONNECT_MS); - await delay(delayMs); + // ref: false so the timer doesn't keep the process alive + await setTimeout(delayMs, undefined, { ref: false }); } }