From 5bd773979666e714dcfa906b5ea88cbae2739db8 Mon Sep 17 00:00:00 2001 From: ndycode <405533+ndycode@users.noreply.github.com> Date: Sun, 22 Mar 2026 12:44:24 +0800 Subject: [PATCH 1/2] test: cover benchmark dashboard renderer --- .../benchmark-render-dashboard-script.test.ts | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 test/benchmark-render-dashboard-script.test.ts diff --git a/test/benchmark-render-dashboard-script.test.ts b/test/benchmark-render-dashboard-script.test.ts new file mode 100644 index 00000000..8b9d9209 --- /dev/null +++ b/test/benchmark-render-dashboard-script.test.ts @@ -0,0 +1,91 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +const scriptPath = path.resolve( + process.cwd(), + "scripts", + "benchmark-render-dashboard.mjs", +); +const tempRoots: string[] = []; + +afterEach(() => { + while (tempRoots.length > 0) { + const root = tempRoots.pop(); + if (root) { + rmSync(root, { recursive: true, force: true }); + } + } +}); + +describe("benchmark render dashboard script", () => { + it("renders HTML from a minimal summary file", () => { + const root = mkdtempSync(path.join(tmpdir(), "bench-render-")); + tempRoots.push(root); + const inputPath = path.join(root, "summary.json"); + const outputPath = path.join(root, "dashboard.html"); + + writeFileSync( + inputPath, + JSON.stringify( + { + meta: { + generatedAt: "2026-03-22T00:00:00.000Z", + preset: "codex-core", + models: ["gpt-5-codex"], + tasks: ["task-1"], + modes: ["patch", "replace", "hashline", "hashline_v2"], + runCount: 1, + warmupCount: 0, + }, + rows: [ + { + modelId: "gpt-5-codex", + displayName: "GPT-5 Codex", + modes: { + patch: { + accuracyPct: 90, + wallMsP50: 1000, + tokensTotalP50: 100, + }, + replace: { + accuracyPct: 85, + wallMsP50: 1100, + tokensTotalP50: 90, + }, + hashline: { + accuracyPct: 88, + wallMsP50: 1050, + tokensTotalP50: 95, + }, + hashline_v2: { + accuracyPct: 92, + wallMsP50: 980, + tokensTotalP50: 80, + }, + }, + }, + ], + failures: [], + }, + null, + 2, + ), + "utf8", + ); + + const result = spawnSync( + process.execPath, + [scriptPath, `--input=${inputPath}`, `--output=${outputPath}`], + { encoding: "utf8" }, + ); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("Dashboard written:"); + const html = readFileSync(outputPath, "utf8"); + expect(html).toContain("Code Edit Format Benchmark"); + expect(html).toContain("GPT-5 Codex"); + }); +}); From 5c5e80df48e9203c7e21a7ae12b902837944fc52 Mon Sep 17 00:00:00 2001 From: ndycode <405533+ndycode@users.noreply.github.com> Date: Sun, 22 Mar 2026 13:08:00 +0800 Subject: [PATCH 2/2] test: harden benchmark dashboard script coverage --- .../benchmark-render-dashboard-script.test.ts | 231 +++++++++++++----- 1 file changed, 172 insertions(+), 59 deletions(-) diff --git a/test/benchmark-render-dashboard-script.test.ts b/test/benchmark-render-dashboard-script.test.ts index 8b9d9209..19e1412a 100644 --- a/test/benchmark-render-dashboard-script.test.ts +++ b/test/benchmark-render-dashboard-script.test.ts @@ -1,5 +1,6 @@ import { spawnSync } from "node:child_process"; -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { rm } from "node:fs/promises"; +import { mkdtempSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -11,81 +12,193 @@ const scriptPath = path.resolve( ); const tempRoots: string[] = []; -afterEach(() => { +async function removeWithRetry( + targetPath: string, + options: { recursive?: boolean; force?: boolean }, +): Promise { + const retryableCodes = new Set(["ENOTEMPTY", "EPERM", "EBUSY"]); + const maxAttempts = 6; + + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + try { + await rm(targetPath, options); + return; + } catch (error) { + const code = + error && + typeof error === "object" && + "code" in error && + typeof error.code === "string" + ? error.code + : undefined; + if (!code || !retryableCodes.has(code) || attempt === maxAttempts) { + throw error; + } + await new Promise((resolve) => setTimeout(resolve, attempt * 50)); + } + } +} + +function createSummaryFixture() { + return { + meta: { + generatedAt: "2026-03-22T00:00:00.000Z", + preset: "codex-core", + models: ["gpt-5-codex"], + tasks: ["task-1"], + modes: ["patch", "replace", "hashline", "hashline_v2"], + runCount: 1, + warmupCount: 0, + }, + rows: [ + { + modelId: "gpt-5-codex", + displayName: "GPT-5 Codex", + modes: { + patch: { + accuracyPct: 90, + wallMsP50: 1000, + tokensTotalP50: 100, + }, + replace: { + accuracyPct: 85, + wallMsP50: 1100, + tokensTotalP50: 90, + }, + hashline: { + accuracyPct: 88, + wallMsP50: 1050, + tokensTotalP50: 95, + }, + hashline_v2: { + accuracyPct: 92, + wallMsP50: 980, + tokensTotalP50: 80, + }, + }, + }, + ], + failures: [], + }; +} + +function createTempRoot(suffix = ""): string { + const root = mkdtempSync(path.join(tmpdir(), `bench-render${suffix}-`)); + tempRoots.push(root); + return root; +} + +function writeSummary(inputPath: string): void { + writeFileSync(inputPath, JSON.stringify(createSummaryFixture(), null, 2), "utf8"); +} + +function runRenderDashboard(args: string[]) { + return spawnSync(process.execPath, [scriptPath, ...args], { + encoding: "utf8", + timeout: 10_000, + }); +} + +afterEach(async () => { while (tempRoots.length > 0) { const root = tempRoots.pop(); if (root) { - rmSync(root, { recursive: true, force: true }); + await removeWithRetry(root, { recursive: true, force: true }); } } }); describe("benchmark render dashboard script", () => { it("renders HTML from a minimal summary file", () => { - const root = mkdtempSync(path.join(tmpdir(), "bench-render-")); - tempRoots.push(root); + const root = createTempRoot(); const inputPath = path.join(root, "summary.json"); const outputPath = path.join(root, "dashboard.html"); - writeFileSync( - inputPath, - JSON.stringify( - { - meta: { - generatedAt: "2026-03-22T00:00:00.000Z", - preset: "codex-core", - models: ["gpt-5-codex"], - tasks: ["task-1"], - modes: ["patch", "replace", "hashline", "hashline_v2"], - runCount: 1, - warmupCount: 0, - }, - rows: [ - { - modelId: "gpt-5-codex", - displayName: "GPT-5 Codex", - modes: { - patch: { - accuracyPct: 90, - wallMsP50: 1000, - tokensTotalP50: 100, - }, - replace: { - accuracyPct: 85, - wallMsP50: 1100, - tokensTotalP50: 90, - }, - hashline: { - accuracyPct: 88, - wallMsP50: 1050, - tokensTotalP50: 95, - }, - hashline_v2: { - accuracyPct: 92, - wallMsP50: 980, - tokensTotalP50: 80, - }, - }, - }, - ], - failures: [], - }, - null, - 2, - ), - "utf8", - ); - - const result = spawnSync( - process.execPath, - [scriptPath, `--input=${inputPath}`, `--output=${outputPath}`], - { encoding: "utf8" }, - ); + writeSummary(inputPath); + const result = runRenderDashboard([ + `--input=${inputPath}`, + `--output=${outputPath}`, + ]); + + expect(result.error).toBeUndefined(); expect(result.status).toBe(0); + expect(result.stderr).toBe(""); expect(result.stdout).toContain("Dashboard written:"); const html = readFileSync(outputPath, "utf8"); expect(html).toContain("Code Edit Format Benchmark"); expect(html).toContain("GPT-5 Codex"); }); + + it("renders HTML when input and output paths contain spaces", () => { + const root = createTempRoot(" spaces"); + const spacedDir = path.join(root, "with spaces"); + mkdirSync(spacedDir, { recursive: true }); + const inputPath = path.join(spacedDir, "summary file.json"); + const outputPath = path.join(spacedDir, "dashboard output.html"); + + writeSummary(inputPath); + + const result = runRenderDashboard([ + `--input=${inputPath}`, + `--output=${outputPath}`, + ]); + + expect(result.error).toBeUndefined(); + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(result.stdout).toContain("Dashboard written:"); + expect(readFileSync(outputPath, "utf8")).toContain("GPT-5 Codex"); + }); + + it("fails with stderr when the input file is missing", () => { + const root = createTempRoot(); + const inputPath = path.join(root, "missing-summary.json"); + const outputPath = path.join(root, "dashboard.html"); + + const result = runRenderDashboard([ + `--input=${inputPath}`, + `--output=${outputPath}`, + ]); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("Render failed:"); + expect(result.stderr.toLowerCase()).toContain("no such file"); + expect(result.stdout).not.toContain("Dashboard written:"); + }); + + it("fails with stderr when the summary json is malformed", () => { + const root = createTempRoot(); + const inputPath = path.join(root, "summary.json"); + const outputPath = path.join(root, "dashboard.html"); + + writeFileSync(inputPath, "{ not-valid-json", "utf8"); + + const result = runRenderDashboard([ + `--input=${inputPath}`, + `--output=${outputPath}`, + ]); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("Render failed:"); + expect(result.stderr.toLowerCase()).toContain("json"); + expect(result.stdout).not.toContain("Dashboard written:"); + }); + + it("fails with stderr when the output directory does not exist", () => { + const root = createTempRoot(); + const inputPath = path.join(root, "summary.json"); + const outputPath = path.join(root, "missing", "dashboard.html"); + + writeSummary(inputPath); + + const result = runRenderDashboard([ + `--input=${inputPath}`, + `--output=${outputPath}`, + ]); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("Render failed:"); + expect(result.stdout).not.toContain("Dashboard written:"); + }); });