From f8144c2b887684c98efe16e837337f5803580409 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 5 Dec 2025 08:34:17 +0000 Subject: [PATCH 1/3] fix(k6): record metrics inside callbacks to handle interrupted sessions The k6 WebSocket test was showing all zeros for most metrics because: 1. wsConnectionSuccess was recorded after ws.connect() returns, which only happens after the socket closes 2. When sessions are interrupted (test ends before SESSION_DURATION_MS), the code after ws.connect() never runs Changes: - Move wsConnectionSuccess.add(1) inside socket.on('open') callback - Record failure (wsConnectionSuccess.add(0)) in error handler and fallback - Fix vus_max to use metrics.vus_max or fallback to VUS constant Co-Authored-By: yujonglee --- apps/k6/scripts/stt-live.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/apps/k6/scripts/stt-live.js b/apps/k6/scripts/stt-live.js index c6100d991..9f2def328 100644 --- a/apps/k6/scripts/stt-live.js +++ b/apps/k6/scripts/stt-live.js @@ -57,10 +57,13 @@ function runSession(data) { const startTime = Date.now(); let firstTranscriptTime = null; let loopCount = 0; + let connectionOpened = false; const res = ws.connect(url, params, function (socket) { socket.on("open", () => { + connectionOpened = true; wsConnections.add(1); + wsConnectionSuccess.add(1); const audioBytes = new Uint8Array(data.audioData); let offset = 0; @@ -102,6 +105,9 @@ function runSession(data) { socket.on("error", (e) => { if (e.error() !== "websocket: close sent") { wsErrors.add(1); + if (!connectionOpened) { + wsConnectionSuccess.add(0); + } console.log(`[VU ${__VU}] Error: ${e.error()}`); } }); @@ -116,14 +122,15 @@ function runSession(data) { }, SESSION_DURATION_MS); }); - const success = res && res.status === 101; - wsConnectionSuccess.add(success ? 1 : 0); + if (!connectionOpened) { + wsConnectionSuccess.add(0); + } check(res, { "WebSocket upgrade successful": (r) => r && r.status === 101, }); - return success; + return connectionOpened; } export default function (data) { @@ -146,7 +153,7 @@ export function handleSummary(data) { const report = { timestamp: new Date().toISOString(), duration_ms: data.state.testRunDurationMs, - vus_max: data.state.vusMax, + vus_max: data.metrics.vus_max?.values?.max || VUS, github: GITHUB_RUN_ID ? { run_id: GITHUB_RUN_ID, From e7b7a1bccda2b0342f76ccc2b3b9ce2e25243a5e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 5 Dec 2025 08:54:05 +0000 Subject: [PATCH 2/3] feat(k6): add message count metrics (sent/received/audio chunks) Track WebSocket message activity: - ws_messages_sent: total outgoing messages (audio + keepalive) - ws_messages_received: total incoming messages - ws_audio_chunks_sent: audio chunks specifically These metrics are recorded inside callbacks so they work even when sessions are interrupted before completing. Co-Authored-By: yujonglee --- apps/k6/scripts/stt-live.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/apps/k6/scripts/stt-live.js b/apps/k6/scripts/stt-live.js index 9f2def328..39c43cbfe 100644 --- a/apps/k6/scripts/stt-live.js +++ b/apps/k6/scripts/stt-live.js @@ -17,6 +17,9 @@ const wsReconnects = new Counter("ws_reconnects"); const wsConnectionDuration = new Trend("ws_connection_duration"); const wsFirstTranscriptLatency = new Trend("ws_first_transcript_latency"); const wsConnectionSuccess = new Rate("ws_connection_success"); +const wsMessagesSent = new Counter("ws_messages_sent"); +const wsMessagesReceived = new Counter("ws_messages_received"); +const wsAudioChunksSent = new Counter("ws_audio_chunks_sent"); const TEST_DURATION = __ENV.TEST_DURATION || "1h"; const VUS = parseInt(__ENV.VUS) || 30; @@ -73,6 +76,8 @@ function runSession(data) { const end = Math.min(offset + CHUNK_SIZE, audioBytes.length); const chunk = audioBytes.slice(offset, end); socket.sendBinary(chunk.buffer); + wsMessagesSent.add(1); + wsAudioChunksSent.add(1); offset = end; } else { offset = 0; @@ -82,10 +87,12 @@ function runSession(data) { socket.setInterval(() => { socket.send(JSON.stringify({ type: "KeepAlive" })); + wsMessagesSent.add(1); }, 3000); }); socket.on("message", (msg) => { + wsMessagesReceived.add(1); try { const response = JSON.parse(msg); @@ -179,6 +186,12 @@ export function handleSummary(data) { avg_ms: data.metrics.ws_connection_duration?.values?.avg || 0, p95_ms: data.metrics.ws_connection_duration?.values?.["p(95)"] || 0, }, + messages: { + sent: data.metrics.ws_messages_sent?.values?.count || 0, + received: data.metrics.ws_messages_received?.values?.count || 0, + audio_chunks_sent: + data.metrics.ws_audio_chunks_sent?.values?.count || 0, + }, checks_passed_rate: data.metrics.checks?.values?.rate || 0, }, server: flyMetrics, @@ -270,6 +283,11 @@ function textSummary(report) { ` Received: ${c.transcripts.received}`, ` First Latency (avg): ${c.transcripts.first_latency_avg_ms.toFixed(0)}ms`, "", + "Messages:", + ` Sent: ${c.messages.sent}`, + ` Received: ${c.messages.received}`, + ` Audio Chunks: ${c.messages.audio_chunks_sent}`, + "", "Connection Duration:", ` Avg: ${formatDuration(c.connection_duration.avg_ms)}`, ` P95: ${formatDuration(c.connection_duration.p95_ms)}`, From 6bf3bd20d668e43f44c8272584afb0c4b24954ca Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 5 Dec 2025 09:09:50 +0000 Subject: [PATCH 3/3] chore: trigger CI re-run Co-Authored-By: yujonglee