diff --git a/e2e/e2e-utils/src/hmrIdle.ts b/e2e/e2e-utils/src/hmrIdle.ts new file mode 100644 index 00000000000..782b46dc776 --- /dev/null +++ b/e2e/e2e-utils/src/hmrIdle.ts @@ -0,0 +1,315 @@ +import type { Page } from '@playwright/test' + +export interface HmrObserverEvent { + kind: string + detail: string + at: number +} + +export interface HmrObserverState { + lastActivityAt: number + installedAt: number + events: Array +} + +declare global { + interface Window { + __HMR_OBSERVER_INSTALLED__?: boolean + __HMR_OBSERVER__?: HmrObserverState + } +} + +/** + * Bundler-agnostic HMR observability for e2e tests. + * + * Why this exists: + * - HMR e2e tests rapidly write source files between assertions/tests. + * - On slow CI machines the dev server's file watcher can: + * * coalesce two writes into one HMR batch (so the second edit appears + * to "vanish") + * * emit a full-page program reload mid-test + * * still be processing the previous test's restore write when the + * current test starts editing + * - Locally these races almost never happen because the watcher latency is + * smaller than the spacing between writes. + * - The fix is to give tests a deterministic way to wait for the dev server + * to be *idle* (no in-flight HMR messages for some quiet window) and to + * wait for a *specific* HMR event after a write. + * + * Implementation: + * We hook `WebSocket` at page-init time and record every incoming text frame + * into a global ring buffer plus a "last activity" timestamp. Both Vite and + * Rsbuild/Rspack push HMR notifications over WebSocket, so this works for + * either toolchain without any app-side instrumentation. + * + * We also record full-page navigations (Vite "program reload"/page reload, + * Rsbuild full reload) as activity so the idle barrier covers them too. + */ + +function hmrObserverInitScript() { + if (window.__HMR_OBSERVER_INSTALLED__) return + window.__HMR_OBSERVER_INSTALLED__ = true + + const state: HmrObserverState = { + lastActivityAt: 0, + events: [], + installedAt: Date.now(), + } + window.__HMR_OBSERVER__ = state + + function record(kind: string, detail: string) { + const now = Date.now() + state.lastActivityAt = now + if (state.events.length > 200) state.events.shift() + state.events.push({ kind: kind, detail: detail, at: now }) + } + + // Mark the page itself loading as activity. Captures Vite "program reload" + // and any other full-page reloads. + record('navigate', location.pathname + location.search) + + const NativeWebSocket = window.WebSocket + const PatchedWebSocket = class extends NativeWebSocket { + constructor(url: string | URL, protocols?: string | Array) { + if (protocols === undefined) { + super(url) + } else { + super(url, protocols) + } + + try { + record('ws:open', String(url)) + this.addEventListener('message', function (ev) { + let detail = '' + try { + detail = + typeof ev.data === 'string' ? ev.data.slice(0, 200) : '[binary]' + } catch { + detail = '[unreadable]' + } + record('ws:message', detail) + }) + this.addEventListener('close', function () { + record('ws:close', String(url)) + }) + this.addEventListener('error', function () { + record('ws:error', String(url)) + }) + } catch { + // never let our instrumentation break the page + } + } + } + window.WebSocket = PatchedWebSocket + + // Vite-specific: listen for HMR lifecycle events when available. Rsbuild's + // client does not expose equivalent window lifecycle events, so Rsbuild is + // covered by the generic WebSocket message observer above. + for (const evt of [ + 'vite:beforeUpdate', + 'vite:afterUpdate', + 'vite:beforeFullReload', + 'vite:beforePrune', + 'vite:invalidate', + 'vite:error', + 'vite:ws:disconnect', + 'vite:ws:connect', + ]) { + window.addEventListener(evt, function (ev) { + let detail = '' + try { + detail = JSON.stringify( + ev instanceof CustomEvent ? ev.detail : null, + ).slice(0, 200) + } catch { + detail = '[unserializable]' + } + record(evt, detail) + }) + } +} + +const HMR_OBSERVER_INIT_SCRIPT = `(${hmrObserverInitScript.toString()})()` + +/** + * Installs the HMR observer for every page in the given context/test. + * Must be called before navigation (e.g. inside a beforeEach via the + * provided fixture), otherwise the first WebSocket may be missed. + */ +export async function installHmrObserver(page: Page) { + await page.addInitScript({ content: HMR_OBSERVER_INIT_SCRIPT }) +} + +async function readObserver(page: Page): Promise { + return await page.evaluate(() => { + const s = window.__HMR_OBSERVER__ + if (!s) return null + return { + lastActivityAt: s.lastActivityAt, + installedAt: s.installedAt, + events: s.events.slice(-50), + } + }) +} + +export async function ensureHmrObserver(page: Page): Promise { + const observer = await readObserver(page).catch(() => null) + if (observer) return + + await page.evaluate(HMR_OBSERVER_INIT_SCRIPT) +} + +export interface WaitForHmrIdleOptions { + /** + * The quiet window the dev server must be silent for, in ms. Defaults to + * 750ms which empirically covers Vite's full-reload + Rsbuild's rebuild + * settle time on slow CI workers. + */ + quietWindowMs?: number + /** + * Maximum total time to wait. Defaults to 20s. + */ + timeoutMs?: number + /** + * Polling interval. Defaults to 100ms. + */ + pollIntervalMs?: number +} + +/** + * Waits until the dev server has been idle (no HMR/WebSocket activity and no + * navigations) for at least `quietWindowMs`. Use this between tests and after + * file restores to ensure HMR events from previous edits don't leak into the + * next assertion. + */ +export async function waitForHmrIdle( + page: Page, + opts: WaitForHmrIdleOptions = {}, +): Promise { + const quietWindowMs = opts.quietWindowMs ?? 750 + const timeoutMs = opts.timeoutMs ?? 20_000 + const pollIntervalMs = opts.pollIntervalMs ?? 100 + + const deadline = Date.now() + timeoutMs + let sawObserver = false + + while (Date.now() < deadline) { + const observer = await readObserver(page).catch(() => null) + if (observer) { + sawObserver = true + const sinceLast = Date.now() - observer.lastActivityAt + if (sinceLast >= quietWindowMs) return + } + await page.waitForTimeout(pollIntervalMs) + } + + if (!sawObserver) { + throw new Error( + 'waitForHmrIdle: HMR observer is not installed on the current page. ' + + 'Call installHmrObserver before navigation or ensureHmrObserver after navigation.', + ) + } + + const observer = await readObserver(page).catch(() => null) + const lastEvents = observer?.events.slice(-5) ?? [] + throw new Error( + `waitForHmrIdle: dev server did not become idle within ${timeoutMs}ms ` + + `(quietWindowMs=${quietWindowMs}). Last events: ` + + JSON.stringify(lastEvents), + ) +} + +export interface WaitForHmrEventOptions { + /** + * Only consider events that occurred after this timestamp (ms since epoch). + * Use the timestamp captured immediately before triggering the edit. + */ + since: number + /** + * Predicate matched against each event's stringified detail. The first event + * (after `since`) whose detail includes/matches this needle resolves the + * promise. + */ + match: string | RegExp + kind?: string | RegExp + /** + * Maximum time to wait. Defaults to 20s. + */ + timeoutMs?: number + /** + * Polling interval. Defaults to 100ms. + */ + pollIntervalMs?: number +} + +/** + * Waits for a specific HMR event matching `match` to appear after `since`. + * Useful to confirm an edit actually produced an HMR update for the expected + * file before asserting on DOM. + */ +export async function waitForHmrEvent( + page: Page, + opts: WaitForHmrEventOptions, +): Promise { + const timeoutMs = opts.timeoutMs ?? 20_000 + const pollIntervalMs = opts.pollIntervalMs ?? 100 + const deadline = Date.now() + timeoutMs + let sawObserver = false + + const matches = (detail: string) => + typeof opts.match === 'string' + ? detail.includes(opts.match) + : opts.match.test(detail) + + const matchesKind = (kind: string) => { + if (!opts.kind) return true + return typeof opts.kind === 'string' + ? kind === opts.kind + : opts.kind.test(kind) + } + + while (Date.now() < deadline) { + const observer = await readObserver(page).catch(() => null) + if (observer) { + sawObserver = true + for (const evt of observer.events) { + if ( + evt.at >= opts.since && + matchesKind(evt.kind) && + matches(evt.detail) + ) { + return + } + } + } + await page.waitForTimeout(pollIntervalMs) + } + + if (!sawObserver) { + throw new Error( + 'waitForHmrEvent: HMR observer is not installed on the current page. ' + + 'Call installHmrObserver before navigation or ensureHmrObserver after navigation.', + ) + } + + const observer = await readObserver(page).catch(() => null) + const lastEvents = observer?.events.slice(-10) ?? [] + throw new Error( + `waitForHmrEvent: no event matching ${String(opts.match)} since ` + + `${opts.since} within ${timeoutMs}ms. Recent events: ` + + JSON.stringify(lastEvents), + ) +} + +/** + * Returns the timestamp of the most recent HMR-related activity observed + * by the page, or 0 if the observer has not yet recorded anything. + */ +export async function getHmrLastActivityAt(page: Page): Promise { + const observer = await readObserver(page).catch(() => null) + return observer?.lastActivityAt ?? 0 +} + +export async function getHmrObserverTime(page: Page): Promise { + return await page.evaluate(() => Date.now()) +} diff --git a/e2e/e2e-utils/src/index.ts b/e2e/e2e-utils/src/index.ts index ec6affdb56c..7ad3a802b52 100644 --- a/e2e/e2e-utils/src/index.ts +++ b/e2e/e2e-utils/src/index.ts @@ -6,3 +6,17 @@ export { e2eStartDummyServer, e2eStopDummyServer } from './e2eSetupTeardown' export { preOptimizeDevServer, waitForServer } from './devServerWarmup' export type { Post } from './posts' export { collectBrowserErrors, test } from './fixture' +export { + ensureHmrObserver, + getHmrLastActivityAt, + getHmrObserverTime, + installHmrObserver, + waitForHmrEvent, + waitForHmrIdle, +} from './hmrIdle' +export type { + HmrObserverEvent, + HmrObserverState, + WaitForHmrEventOptions, + WaitForHmrIdleOptions, +} from './hmrIdle' diff --git a/e2e/react-start/hmr/tests/app.spec.ts b/e2e/react-start/hmr/tests/app.spec.ts index bbaff95bc13..36be9843669 100644 --- a/e2e/react-start/hmr/tests/app.spec.ts +++ b/e2e/react-start/hmr/tests/app.spec.ts @@ -1,7 +1,14 @@ -import { expect } from '@playwright/test' -import { test } from '@tanstack/router-e2e-utils' import { readFile, writeFile } from 'node:fs/promises' import path from 'node:path' +import { expect } from '@playwright/test' +import { + ensureHmrObserver, + getHmrObserverTime, + installHmrObserver, + test, + waitForHmrEvent, + waitForHmrIdle, +} from '@tanstack/router-e2e-utils' import type { Page } from '@playwright/test' const whitelistErrors = [ @@ -307,6 +314,41 @@ async function replaceRouteText( await writeFile(filePath, source.replace(from, to)) } +/** + * The CI flake fix: + * + * Vite/Rsbuild file watchers can coalesce closely-spaced writes into one HMR + * batch on slow CI machines. When that happens, the second write's HMR event + * is dropped, the page never sees the new code, and the assertion times out + * polling stale "baseline" content for 20s. + * + * To make the writes deterministic, we wait for the dev server to be idle + * (no in-flight HMR/WebSocket activity) BEFORE writing. That guarantees the + * watcher will treat our write as a fresh event rather than coalescing it + * with leftover work from the previous test or restore. + */ +async function settleBeforeWrite(page: Page, quietWindowMs = 750) { + await ensureHmrObserver(page) + await waitForHmrIdle(page, { quietWindowMs, timeoutMs: 20_000 }) +} + +async function waitForHmrAfterWrite(page: Page, since: number) { + const hmrActivityKind = + /^(ws:message|vite:beforeUpdate|vite:afterUpdate|vite:beforeFullReload|navigate)$/ + + await waitForHmrEvent(page, { + since, + // Vite emits typed window events with structured detail. Rsbuild/Rspack + // does not expose equivalent browser lifecycle events, so WebSocket frames + // are the common cross-toolchain signal. Full reloads are recorded as a + // fresh navigate event by the init script on the new document. + kind: hmrActivityKind, + match: /.+/, + timeoutMs: 20_000, + }) + await waitForHmrIdle(page, { quietWindowMs: 750, timeoutMs: 20_000 }) +} + async function replaceRouteTextAndWait( page: Page, routeFileKey: RouteFileKey, @@ -314,7 +356,10 @@ async function replaceRouteTextAndWait( to: string, assertion: () => Promise, ) { + await settleBeforeWrite(page) + const since = await getHmrObserverTime(page) await replaceRouteText(routeFileKey, from, to) + await waitForHmrAfterWrite(page, since) await assertion() } @@ -333,9 +378,12 @@ async function rewriteRouteFile( throw new Error(`Expected ${filePath} to change during rewrite`) } + await settleBeforeWrite(page) + const since = await getHmrObserverTime(page) // Even a no-op write is useful for tests that need to force the dev server // to reconcile a stale in-memory module with the current file contents. await writeFile(filePath, updated) + await waitForHmrAfterWrite(page, since) await assertion() } @@ -551,6 +599,13 @@ test.describe('react-start hmr', () => { test.use({ whitelistErrors }) test.beforeEach(async ({ page }) => { + // Install the HMR/WebSocket observer BEFORE any navigation so we can + // detect when the dev server is idle between writes. This is the + // single biggest contributor to CI flakiness — without an idle barrier, + // closely-spaced file writes get coalesced by the watcher and the + // second HMR event is silently dropped. + await installHmrObserver(page) + await capturePromise const pendingRouteKeys = Array.from(routeKeysPendingRestoreCheck) const restoredRouteKeys = await restoreRouteFiles(pendingRouteKeys) @@ -564,6 +619,14 @@ test.describe('react-start hmr', () => { for (const routeFileKey of routeKeysToCheck) { await waitForRestoredRouteFile(page, routeFileKey) } + + // After restoring files, give the dev server a chance to drain any HMR + // work triggered by those writes. Without this barrier, the test's first + // edit can race with leftover HMR processing and the watcher coalesces + // both into one event (dropping the test's edit). A 1s quiet window is + // generous enough for slow CI workers and adds <1s to the total run. + await ensureHmrObserver(page) + await waitForHmrIdle(page, { quietWindowMs: 1_000, timeoutMs: 20_000 }) }) test.afterEach(async () => { @@ -1140,11 +1203,14 @@ test.describe('react-start hmr', () => { ) await expect(page.getByTestId('server-fn-hmr-error')).toHaveText('none') + await settleBeforeWrite(page) + const clientOnlySince = await getHmrObserverTime(page) await replaceRouteText( 'serverFnHmrFactory', "createServerOnlyFn\nexport const serverFnHmrMarker = 'server-fn-hmr-baseline'", "createClientOnlyFn\nexport const serverFnHmrMarker = 'server-fn-hmr-client-only'", ) + await waitForHmrAfterWrite(page, clientOnlySince) await reloadPageAndWaitForText( page, '/server-fn-hmr', @@ -1158,11 +1224,14 @@ test.describe('react-start hmr', () => { 'createClientOnlyFn() functions can only be called on the client!', ) + await settleBeforeWrite(page) + const baselineSince = await getHmrObserverTime(page) await replaceRouteText( 'serverFnHmrFactory', "createClientOnlyFn\nexport const serverFnHmrMarker = 'server-fn-hmr-client-only'", "createServerOnlyFn\nexport const serverFnHmrMarker = 'server-fn-hmr-baseline'", ) + await waitForHmrAfterWrite(page, baselineSince) await reloadPageAndWaitForText( page, '/server-fn-hmr',