From f08690e49b57bb17f164809d4e5d6de3af09fb0f Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 12:14:45 +0000 Subject: [PATCH 1/6] feat: Move WebCodecs decoder to Web Worker for better scrcpy video streaming performance To improve scrcpy streams performance when monitoring up to 6 devices concurrently, WebCodecs decoding tasks have been moved off the main thread to a dedicated Web Worker instance for each device. The application now allocates an OffscreenCanvas that is passed to each worker alongside the scrcpy packet streams. The background Web Worker runs the actual \`@yume-chan/scrcpy-decoder-webcodecs\` logic, resulting in vastly improved stability and responsiveness on the main thread. Old active Web Workers are automatically tracked and properly terminated when devices disconnect, streams switch codec, or components unmount. Co-authored-by: RoiArthurB <16764085+RoiArthurB@users.noreply.github.com> --- public/scrcpy/worker.js | 0 .../WebSocketManager/VideoStreamManager.tsx | 94 +++++++++++-------- src/workers/scrcpyDecoder.ts | 36 +++++++ tsconfig.app.json | 2 +- 4 files changed, 94 insertions(+), 38 deletions(-) create mode 100644 public/scrcpy/worker.js create mode 100644 src/workers/scrcpyDecoder.ts diff --git a/public/scrcpy/worker.js b/public/scrcpy/worker.js new file mode 100644 index 0000000..e69de29 diff --git a/src/components/WebSocketManager/VideoStreamManager.tsx b/src/components/WebSocketManager/VideoStreamManager.tsx index dab36bd..9e41a22 100644 --- a/src/components/WebSocketManager/VideoStreamManager.tsx +++ b/src/components/WebSocketManager/VideoStreamManager.tsx @@ -1,11 +1,5 @@ import { useEffect, useState, useMemo, useRef } from "react"; import PlayerScreenCanvas from "./PlayerScreenCanvas.tsx"; -import { - VideoFrameRenderer, - WebGLVideoFrameRenderer, - BitmapVideoFrameRenderer, - WebCodecsVideoDecoder, -} from "@yume-chan/scrcpy-decoder-webcodecs"; import { getLogger } from "@logtape/logtape"; import { ScrcpyMediaStreamPacket, ScrcpyVideoCodecId } from "@yume-chan/scrcpy"; const host: string = window.location.hostname; @@ -41,19 +35,6 @@ const deserializeData = (serializedData: string) => { } }; -function createVideoFrameRenderer(): VideoFrameRenderer { - - if (WebGLVideoFrameRenderer.isSupported) { - logger.debug("[SCRCPY] Using WebGLVideoFrameRenderer"); - return new WebGLVideoFrameRenderer(); - } else { - logger.warn("[SCRCPY] WebGL isn't supported... "); - } - - logger.debug("[SCRCPY] Using fallback BitmapVideoFrameRenderer"); - return new BitmapVideoFrameRenderer(); -} - interface VideoStreamManagerProps { needsInteractivity?: boolean; selectedCanvas?: string; @@ -92,6 +73,8 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V const isDecoderHasConfig = new Map(); // Tracks the codec each decoder was created with — used to detect mid-stream codec changes const streamIsH265 = new Map(); + // Map of worker instances for each stream + const decoderWorkers = useRef>(new Map()); @@ -111,10 +94,7 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V } // Prepare video stream ======================= - const renderer: VideoFrameRenderer = createVideoFrameRenderer(); - - // get the canvas from the renderer (renderer as any is used to ensure ts knows that canvas is a property of the renderer) - const canvas = (renderer as any).canvas as HTMLCanvasElement + const canvas = document.createElement("canvas"); // Catch cases with non IP devices (USB const canvasId: string = @@ -136,6 +116,24 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V } } + const offscreenCanvas = canvas.transferControlToOffscreen(); + + // Clean up any existing worker + if (decoderWorkers.current.has(deviceId)) { + decoderWorkers.current.get(deviceId)?.terminate(); + decoderWorkers.current.delete(deviceId); + } + + // Create a new web worker to handle the stream + const worker = new Worker(new URL("../../workers/scrcpyDecoder.ts", import.meta.url), { type: "module" }); + decoderWorkers.current.set(deviceId, worker); + + worker.addEventListener("message", (e) => { + if (e.data.type === 'sizeChanged') { + logger.debug(`[Scrcpy] Size changed for ${deviceId}: ${e.data.width}x${e.data.height}`); + } + }); + // Create the ReadableStream BEFORE the async codec check. // new ReadableStream() calls start() synchronously, so the real controller is placed // in readableControllers before this function ever suspends at the first await. @@ -152,6 +150,13 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V cancel() { readableControllers.delete(deviceId); isDecoderHasConfig.delete(deviceId); + + // Terminate the worker + if (decoderWorkers.current.has(deviceId)) { + decoderWorkers.current.get(deviceId)?.terminate(); + decoderWorkers.current.delete(deviceId); + } + // Use the canvas already in scope — canvasList captures a stale closure // (React state at render time) so canvasList[deviceId] is always undefined here. canvas.remove(); @@ -173,24 +178,24 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V if (useH265 && !supported.supported) { logger.warn("[Scrcpy-VideoStreamManager] Should decode h265, but not compatible, waiting for new stream to start..."); readableControllers.delete(deviceId); + worker.terminate(); + decoderWorkers.current.delete(deviceId); return; } if (supported.supported || !useH265) { - const decoder = new WebCodecsVideoDecoder({ - codec: useH265 ? ScrcpyVideoCodecId.H265 : ScrcpyVideoCodecId.H264, - renderer: renderer, - // Firefox on Linux has no hardware H264 WebCodecs path; "prefer-software" enables - // the software decoder (OpenH264) and avoids "encoding not supported" errors. - // H265 keeps "no-preference": Chrome only supports H265 via hardware, so forcing - // software would cause "OperationError: Unsupported configuration". - hardwareAcceleration: useH265 ? "no-preference" : "prefer-software", - }); - - // Feed the scrcpy stream to the video decoder - void stream.pipeTo(decoder.writable).catch((err) => { - logger.error("[Scrcpy] Error piping to decoder writable stream: {err}", { err }); - }); + const codec = useH265 ? ScrcpyVideoCodecId.H265 : ScrcpyVideoCodecId.H264; + + // Pass objects and stream to worker + worker.postMessage( + { + codec, + canvas: offscreenCanvas, + stream, + useH265 + }, + [offscreenCanvas, stream] + ); } else { logger.error("[Scrcpy] Error piping to decoder writable stream"); } @@ -223,6 +228,12 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V // Clearing these forces newVideoStream() to recreate the decoder on the next config packet. readableControllers.delete(streamId); isDecoderHasConfig.delete(streamId); + + // Terminate the worker + if (decoderWorkers.current.has(streamId)) { + decoderWorkers.current.get(streamId)?.terminate(); + decoderWorkers.current.delete(streamId); + } } const ws = new WebSocket(`ws://${host}:${port}/stream/${streamId}`); @@ -303,6 +314,13 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V readableControllers.delete(streamId); isDecoderHasConfig.delete(streamId); streamIsH265.delete(streamId); + + // Terminate the worker + if (decoderWorkers.current.has(streamId)) { + decoderWorkers.current.get(streamId)?.terminate(); + decoderWorkers.current.delete(streamId); + } + // Remove canvas from DOM and React state const canvas = canvasRefs.current.get(streamId); if (canvas) { @@ -396,6 +414,8 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V cleanedUp = true; controlSocket?.close(); deviceSockets.forEach(ws => ws.close()); + decoderWorkers.current.forEach(worker => worker.terminate()); + decoderWorkers.current.clear(); }; }, []); diff --git a/src/workers/scrcpyDecoder.ts b/src/workers/scrcpyDecoder.ts new file mode 100644 index 0000000..2e06edc --- /dev/null +++ b/src/workers/scrcpyDecoder.ts @@ -0,0 +1,36 @@ +import { ScrcpyVideoCodecId, type ScrcpyMediaStreamPacket } from "@yume-chan/scrcpy"; +import { + WebGLVideoFrameRenderer, + BitmapVideoFrameRenderer, + WebCodecsVideoDecoder, +} from "@yume-chan/scrcpy-decoder-webcodecs"; + +self.addEventListener("message", (e) => { + const { codec, canvas, stream, useH265 } = e.data as { + codec: ScrcpyVideoCodecId; + canvas: OffscreenCanvas; + stream: ReadableStream; + useH265: boolean; + }; + + let renderer; + if (WebGLVideoFrameRenderer.isSupported) { + renderer = new WebGLVideoFrameRenderer(canvas); + } else { + renderer = new BitmapVideoFrameRenderer(canvas); + } + + const decoder = new WebCodecsVideoDecoder({ + codec: codec, + renderer: renderer, + hardwareAcceleration: useH265 ? "no-preference" : "prefer-software", + }); + + decoder.sizeChanged(({ width, height }) => { + postMessage({ type: 'sizeChanged', width, height }); + }); + + void stream.pipeTo(decoder.writable).catch((err) => { + console.error("[Worker] Error piping to decoder writable stream:", err); + }); +}); diff --git a/tsconfig.app.json b/tsconfig.app.json index 1621296..8d8cbf4 100644 --- a/tsconfig.app.json +++ b/tsconfig.app.json @@ -2,7 +2,7 @@ "compilerOptions": { "composite": true, "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo", - "target": "ES2021", + "target": "ES2022", "useDefineForClassFields": true, "lib": ["ES2021", "DOM", "DOM.Iterable"], "module": "ESNext", From c8a7046a7203ff99e58235bac1c064564f533dd2 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Tue, 17 Mar 2026 06:49:45 +0000 Subject: [PATCH 2/6] feat: Move WebCodecs decoder to Web Worker for better scrcpy video streaming performance To improve scrcpy streams performance when monitoring up to 6 devices concurrently, WebCodecs decoding tasks have been moved off the main thread to a dedicated Web Worker instance for each device. The application now allocates an OffscreenCanvas that is passed to each worker alongside the scrcpy packet streams. The background Web Worker runs the actual \`@yume-chan/scrcpy-decoder-webcodecs\` logic, resulting in vastly improved stability and responsiveness on the main thread. Old active Web Workers are automatically tracked and properly terminated when devices disconnect, streams switch codec, or components unmount. Co-authored-by: RoiArthurB <16764085+RoiArthurB@users.noreply.github.com> From ff2d8bc1d4054883d988d6cd9487f308b7d50681 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Tue, 17 Mar 2026 06:50:54 +0000 Subject: [PATCH 3/6] fix: Address SonarCloud security concern in Web Worker message event handler Check \`e.origin !== self.location.origin\` in \`scrcpyDecoder.ts\` to verify that incoming messages from the main thread originate from a trusted source. Co-authored-by: RoiArthurB <16764085+RoiArthurB@users.noreply.github.com> --- src/workers/scrcpyDecoder.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/workers/scrcpyDecoder.ts b/src/workers/scrcpyDecoder.ts index 2e06edc..ae76bf1 100644 --- a/src/workers/scrcpyDecoder.ts +++ b/src/workers/scrcpyDecoder.ts @@ -6,6 +6,11 @@ import { } from "@yume-chan/scrcpy-decoder-webcodecs"; self.addEventListener("message", (e) => { + // Ensure the message originates from the same origin to mitigate security risks + if (e.origin !== self.location.origin) { + return; + } + const { codec, canvas, stream, useH265 } = e.data as { codec: ScrcpyVideoCodecId; canvas: OffscreenCanvas; From 1fdf7e18c8fb3d61d20da6d133a3b0fe2645d7cc Mon Sep 17 00:00:00 2001 From: RoiArthurB Date: Tue, 17 Mar 2026 14:14:08 +0700 Subject: [PATCH 4/6] fix: Remove spamming worker debug log For some reason, it's spamming a lot for nothing And even if I wanted to leave them, those are ultimately quite useless... --- src/components/WebSocketManager/VideoStreamManager.tsx | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/components/WebSocketManager/VideoStreamManager.tsx b/src/components/WebSocketManager/VideoStreamManager.tsx index 9e41a22..5ec08f1 100644 --- a/src/components/WebSocketManager/VideoStreamManager.tsx +++ b/src/components/WebSocketManager/VideoStreamManager.tsx @@ -128,12 +128,6 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V const worker = new Worker(new URL("../../workers/scrcpyDecoder.ts", import.meta.url), { type: "module" }); decoderWorkers.current.set(deviceId, worker); - worker.addEventListener("message", (e) => { - if (e.data.type === 'sizeChanged') { - logger.debug(`[Scrcpy] Size changed for ${deviceId}: ${e.data.width}x${e.data.height}`); - } - }); - // Create the ReadableStream BEFORE the async codec check. // new ReadableStream() calls start() synchronously, so the real controller is placed // in readableControllers before this function ever suspends at the first await. From 21ec158774ad82ebf518c3b3aef248f4d840a5f8 Mon Sep 17 00:00:00 2001 From: RoiArthurB Date: Tue, 17 Mar 2026 14:19:11 +0700 Subject: [PATCH 5/6] Revert "fix: Address SonarCloud security concern in Web Worker message event handler" This reverts commit ff2d8bc1d4054883d988d6cd9487f308b7d50681. Was breaking data packages reception, and this security concerns isn't applicable in the project :) --- src/workers/scrcpyDecoder.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/workers/scrcpyDecoder.ts b/src/workers/scrcpyDecoder.ts index ae76bf1..2e06edc 100644 --- a/src/workers/scrcpyDecoder.ts +++ b/src/workers/scrcpyDecoder.ts @@ -6,11 +6,6 @@ import { } from "@yume-chan/scrcpy-decoder-webcodecs"; self.addEventListener("message", (e) => { - // Ensure the message originates from the same origin to mitigate security risks - if (e.origin !== self.location.origin) { - return; - } - const { codec, canvas, stream, useH265 } = e.data as { codec: ScrcpyVideoCodecId; canvas: OffscreenCanvas; From 0a683c972e0a327e5915dd0d294db667492cda40 Mon Sep 17 00:00:00 2001 From: RoiArthurB Date: Tue, 17 Mar 2026 14:24:07 +0700 Subject: [PATCH 6/6] chore: Cleanup useless empty file --- public/scrcpy/worker.js | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 public/scrcpy/worker.js diff --git a/public/scrcpy/worker.js b/public/scrcpy/worker.js deleted file mode 100644 index e69de29..0000000