feat: Move WebCodecs decoder to Web Worker for better scrcpy video streaming performance

To improve scrcpy streams performance when monitoring up to 6 devices concurrently, WebCodecs decoding tasks have been moved off the main thread to a dedicated Web Worker instance for each device.
The application now allocates an OffscreenCanvas that is passed to each worker alongside the scrcpy packet streams. The background Web Worker runs the actual \`@yume-chan/scrcpy-decoder-webcodecs\` logic, resulting in vastly improved stability and responsiveness on the main thread.
Old active Web Workers are automatically tracked and properly terminated when devices disconnect, streams switch codec, or components unmount.

Co-authored-by: RoiArthurB <16764085+RoiArthurB@users.noreply.github.com>

Author: google-labs-jules[bot]

public/scrcpy/worker.js

@@ -1,11 +1,5 @@
 import { useEffect, useState, useMemo, useRef } from "react";
 import PlayerScreenCanvas from "./PlayerScreenCanvas.tsx";
-import {
-  VideoFrameRenderer,
-  WebGLVideoFrameRenderer,
-  BitmapVideoFrameRenderer,
-  WebCodecsVideoDecoder,
-} from "@yume-chan/scrcpy-decoder-webcodecs";
 import { getLogger } from "@logtape/logtape";
 import { ScrcpyMediaStreamPacket, ScrcpyVideoCodecId } from "@yume-chan/scrcpy";
 const host: string = window.location.hostname;
@@ -41,19 +35,6 @@ const deserializeData = (serializedData: string) => {
   }
 };
 
-function createVideoFrameRenderer(): VideoFrameRenderer {
-
-  if (WebGLVideoFrameRenderer.isSupported) {
-    logger.debug("[SCRCPY] Using WebGLVideoFrameRenderer");
-    return new WebGLVideoFrameRenderer();
-  } else {
-    logger.warn("[SCRCPY] WebGL isn't supported... ");
-  }
-
-  logger.debug("[SCRCPY] Using fallback BitmapVideoFrameRenderer");
-  return new BitmapVideoFrameRenderer();
-}
-
 interface VideoStreamManagerProps {
   needsInteractivity?: boolean;
   selectedCanvas?: string;
@@ -92,6 +73,8 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V
   const isDecoderHasConfig = new Map<string, boolean>();
   // Tracks the codec each decoder was created with — used to detect mid-stream codec changes
   const streamIsH265 = new Map<string, boolean>();
+  // Map of worker instances for each stream
+  const decoderWorkers = useRef<Map<string, Worker>>(new Map());
 
 
 
@@ -111,10 +94,7 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V
     }
     // Prepare video stream =======================
 
-    const renderer: VideoFrameRenderer = createVideoFrameRenderer();
-
-    // get the canvas from the renderer (renderer as any is used to ensure ts knows that canvas is a property of the renderer)
-    const canvas = (renderer as any).canvas as HTMLCanvasElement
+    const canvas = document.createElement("canvas");
 
     // Catch cases with non IP devices (USB
     const canvasId: string =
@@ -136,6 +116,24 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V
       }
     }
 
+    const offscreenCanvas = canvas.transferControlToOffscreen();
+
+    // Clean up any existing worker
+    if (decoderWorkers.current.has(deviceId)) {
+      decoderWorkers.current.get(deviceId)?.terminate();
+      decoderWorkers.current.delete(deviceId);
+    }
+
+    // Create a new web worker to handle the stream
+    const worker = new Worker(new URL("../../workers/scrcpyDecoder.ts", import.meta.url), { type: "module" });
+    decoderWorkers.current.set(deviceId, worker);
+
+    worker.addEventListener("message", (e) => {
+      if (e.data.type === 'sizeChanged') {
+        logger.debug(`[Scrcpy] Size changed for ${deviceId}: ${e.data.width}x${e.data.height}`);
+      }
+    });
+
     // Create the ReadableStream BEFORE the async codec check.
     // new ReadableStream() calls start() synchronously, so the real controller is placed
     // in readableControllers before this function ever suspends at the first await.
@@ -152,6 +150,13 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V
       cancel() {
         readableControllers.delete(deviceId);
         isDecoderHasConfig.delete(deviceId);
+
+        // Terminate the worker
+        if (decoderWorkers.current.has(deviceId)) {
+          decoderWorkers.current.get(deviceId)?.terminate();
+          decoderWorkers.current.delete(deviceId);
+        }
+
         // Use the canvas already in scope — canvasList captures a stale closure
         // (React state at render time) so canvasList[deviceId] is always undefined here.
         canvas.remove();
@@ -173,24 +178,24 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V
       if (useH265 && !supported.supported) {
         logger.warn("[Scrcpy-VideoStreamManager] Should decode h265, but not compatible, waiting for new stream to start...");
         readableControllers.delete(deviceId);
+        worker.terminate();
+        decoderWorkers.current.delete(deviceId);
         return;
       }
 
       if (supported.supported || !useH265) {
-        const decoder = new WebCodecsVideoDecoder({
-          codec: useH265 ? ScrcpyVideoCodecId.H265 : ScrcpyVideoCodecId.H264,
-          renderer: renderer,
-          // Firefox on Linux has no hardware H264 WebCodecs path; "prefer-software" enables
-          // the software decoder (OpenH264) and avoids "encoding not supported" errors.
-          // H265 keeps "no-preference": Chrome only supports H265 via hardware, so forcing
-          // software would cause "OperationError: Unsupported configuration".
-          hardwareAcceleration: useH265 ? "no-preference" : "prefer-software",
-        });
-
-        // Feed the scrcpy stream to the video decoder
-        void stream.pipeTo(decoder.writable).catch((err) => {
-          logger.error("[Scrcpy] Error piping to decoder writable stream: {err}", { err });
-        });
+        const codec = useH265 ? ScrcpyVideoCodecId.H265 : ScrcpyVideoCodecId.H264;
+
+        // Pass objects and stream to worker
+        worker.postMessage(
+          {
+            codec,
+            canvas: offscreenCanvas,
+            stream,
+            useH265
+          },
+          [offscreenCanvas, stream]
+        );
       } else {
         logger.error("[Scrcpy] Error piping to decoder writable stream");
       }
@@ -223,6 +228,12 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V
         // Clearing these forces newVideoStream() to recreate the decoder on the next config packet.
         readableControllers.delete(streamId);
         isDecoderHasConfig.delete(streamId);
+
+        // Terminate the worker
+        if (decoderWorkers.current.has(streamId)) {
+          decoderWorkers.current.get(streamId)?.terminate();
+          decoderWorkers.current.delete(streamId);
+        }
       }
 
       const ws = new WebSocket(`ws://${host}:${port}/stream/${streamId}`);
@@ -303,6 +314,13 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V
       readableControllers.delete(streamId);
       isDecoderHasConfig.delete(streamId);
       streamIsH265.delete(streamId);
+
+      // Terminate the worker
+      if (decoderWorkers.current.has(streamId)) {
+        decoderWorkers.current.get(streamId)?.terminate();
+        decoderWorkers.current.delete(streamId);
+      }
+
       // Remove canvas from DOM and React state
       const canvas = canvasRefs.current.get(streamId);
       if (canvas) {
@@ -396,6 +414,8 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V
       cleanedUp = true;
       controlSocket?.close();
       deviceSockets.forEach(ws => ws.close());
+      decoderWorkers.current.forEach(worker => worker.terminate());
+      decoderWorkers.current.clear();
     };
   }, []);
 

src/components/WebSocketManager/VideoStreamManager.tsx

@@ -0,0 +1,36 @@
+import { ScrcpyVideoCodecId, type ScrcpyMediaStreamPacket } from "@yume-chan/scrcpy";
+import {
+  WebGLVideoFrameRenderer,
+  BitmapVideoFrameRenderer,
+  WebCodecsVideoDecoder,
+} from "@yume-chan/scrcpy-decoder-webcodecs";
+
+self.addEventListener("message", (e) => {
+  const { codec, canvas, stream, useH265 } = e.data as {
+    codec: ScrcpyVideoCodecId;
+    canvas: OffscreenCanvas;
+    stream: ReadableStream<ScrcpyMediaStreamPacket>;
+    useH265: boolean;
+  };
+
+  let renderer;
+  if (WebGLVideoFrameRenderer.isSupported) {
+    renderer = new WebGLVideoFrameRenderer(canvas);
+  } else {
+    renderer = new BitmapVideoFrameRenderer(canvas);
+  }
+
+  const decoder = new WebCodecsVideoDecoder({
+    codec: codec,
+    renderer: renderer,
+    hardwareAcceleration: useH265 ? "no-preference" : "prefer-software",
+  });
+
+  decoder.sizeChanged(({ width, height }) => {
+    postMessage({ type: 'sizeChanged', width, height });
+  });
+
+  void stream.pipeTo(decoder.writable).catch((err) => {
+    console.error("[Worker] Error piping to decoder writable stream:", err);
+  });
+});

src/workers/scrcpyDecoder.ts

@@ -2,7 +2,7 @@
   "compilerOptions": {
     "composite": true,
     "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo",
-    "target": "ES2021",
+    "target": "ES2022",
     "useDefineForClassFields": true,
     "lib": ["ES2021", "DOM", "DOM.Iterable"],
     "module": "ESNext",