Move WebCodecs decoder to Web Worker for better scrcpy video streaming performance

src/components/WebSocketManager/VideoStreamManager.tsx

@@ -1,11 +1,5 @@
 import { useEffect, useState, useMemo, useRef } from "react";
 import PlayerScreenCanvas from "./PlayerScreenCanvas.tsx";
-import {
-  VideoFrameRenderer,
-  WebGLVideoFrameRenderer,
-  BitmapVideoFrameRenderer,
-  WebCodecsVideoDecoder,
-} from "@yume-chan/scrcpy-decoder-webcodecs";
 import { getLogger } from "@logtape/logtape";
 import { ScrcpyMediaStreamPacket, ScrcpyVideoCodecId } from "@yume-chan/scrcpy";
 const host: string = window.location.hostname;
@@ -41,19 +35,6 @@ const deserializeData = (serializedData: string) => {
   }
 };
 
-function createVideoFrameRenderer(): VideoFrameRenderer {
-
-  if (WebGLVideoFrameRenderer.isSupported) {
-    logger.debug("[SCRCPY] Using WebGLVideoFrameRenderer");
-    return new WebGLVideoFrameRenderer();
-  } else {
-    logger.warn("[SCRCPY] WebGL isn't supported... ");
-  }
-
-  logger.debug("[SCRCPY] Using fallback BitmapVideoFrameRenderer");
-  return new BitmapVideoFrameRenderer();
-}
-
 interface VideoStreamManagerProps {
   needsInteractivity?: boolean;
   selectedCanvas?: string;
@@ -92,6 +73,8 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V
   const isDecoderHasConfig = new Map<string, boolean>();
   // Tracks the codec each decoder was created with — used to detect mid-stream codec changes
   const streamIsH265 = new Map<string, boolean>();
+  // Map of worker instances for each stream
+  const decoderWorkers = useRef<Map<string, Worker>>(new Map());
 
 
 
@@ -111,10 +94,7 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V
     }
     // Prepare video stream =======================
 
-    const renderer: VideoFrameRenderer = createVideoFrameRenderer();
-
-    // get the canvas from the renderer (renderer as any is used to ensure ts knows that canvas is a property of the renderer)
-    const canvas = (renderer as any).canvas as HTMLCanvasElement
+    const canvas = document.createElement("canvas");
 
     // Catch cases with non IP devices (USB
     const canvasId: string =
@@ -136,6 +116,18 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V
       }
     }
 
+    const offscreenCanvas = canvas.transferControlToOffscreen();
+
+    // Clean up any existing worker
+    if (decoderWorkers.current.has(deviceId)) {
+      decoderWorkers.current.get(deviceId)?.terminate();
+      decoderWorkers.current.delete(deviceId);
+    }
+
+    // Create a new web worker to handle the stream
+    const worker = new Worker(new URL("../../workers/scrcpyDecoder.ts", import.meta.url), { type: "module" });
+    decoderWorkers.current.set(deviceId, worker);
+
     // Create the ReadableStream BEFORE the async codec check.
     // new ReadableStream() calls start() synchronously, so the real controller is placed
     // in readableControllers before this function ever suspends at the first await.
@@ -152,6 +144,13 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V
       cancel() {
         readableControllers.delete(deviceId);
         isDecoderHasConfig.delete(deviceId);
+
+        // Terminate the worker
+        if (decoderWorkers.current.has(deviceId)) {
+          decoderWorkers.current.get(deviceId)?.terminate();
+          decoderWorkers.current.delete(deviceId);
+        }
+
         // Use the canvas already in scope — canvasList captures a stale closure
         // (React state at render time) so canvasList[deviceId] is always undefined here.
         canvas.remove();
@@ -173,24 +172,24 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V
       if (useH265 && !supported.supported) {
         logger.warn("[Scrcpy-VideoStreamManager] Should decode h265, but not compatible, waiting for new stream to start...");
         readableControllers.delete(deviceId);
+        worker.terminate();
+        decoderWorkers.current.delete(deviceId);
         return;
       }
 
       if (supported.supported || !useH265) {
-        const decoder = new WebCodecsVideoDecoder({
-          codec: useH265 ? ScrcpyVideoCodecId.H265 : ScrcpyVideoCodecId.H264,
-          renderer: renderer,
-          // Firefox on Linux has no hardware H264 WebCodecs path; "prefer-software" enables
-          // the software decoder (OpenH264) and avoids "encoding not supported" errors.
-          // H265 keeps "no-preference": Chrome only supports H265 via hardware, so forcing
-          // software would cause "OperationError: Unsupported configuration".
-          hardwareAcceleration: useH265 ? "no-preference" : "prefer-software",
-        });
-
-        // Feed the scrcpy stream to the video decoder
-        void stream.pipeTo(decoder.writable).catch((err) => {
-          logger.error("[Scrcpy] Error piping to decoder writable stream: {err}", { err });
-        });
+        const codec = useH265 ? ScrcpyVideoCodecId.H265 : ScrcpyVideoCodecId.H264;
+
+        // Pass objects and stream to worker
+        worker.postMessage(
+          {
+            codec,
+            canvas: offscreenCanvas,
+            stream,
+            useH265
+          },
+          [offscreenCanvas, stream]
+        );
       } else {
         logger.error("[Scrcpy] Error piping to decoder writable stream");
       }
@@ -223,6 +222,12 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V
         // Clearing these forces newVideoStream() to recreate the decoder on the next config packet.
         readableControllers.delete(streamId);
         isDecoderHasConfig.delete(streamId);
+
+        // Terminate the worker
+        if (decoderWorkers.current.has(streamId)) {
+          decoderWorkers.current.get(streamId)?.terminate();
+          decoderWorkers.current.delete(streamId);
+        }
       }
 
       const ws = new WebSocket(`ws://${host}:${port}/stream/${streamId}`);
@@ -303,6 +308,13 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V
       readableControllers.delete(streamId);
       isDecoderHasConfig.delete(streamId);
       streamIsH265.delete(streamId);
+
+      // Terminate the worker
+      if (decoderWorkers.current.has(streamId)) {
+        decoderWorkers.current.get(streamId)?.terminate();
+        decoderWorkers.current.delete(streamId);
+      }
+
       // Remove canvas from DOM and React state
       const canvas = canvasRefs.current.get(streamId);
       if (canvas) {
@@ -396,6 +408,8 @@ const VideoStreamManager = ({ needsInteractivity, selectedCanvas, hideInfos }: V
       cleanedUp = true;
       controlSocket?.close();
       deviceSockets.forEach(ws => ws.close());
+      decoderWorkers.current.forEach(worker => worker.terminate());
+      decoderWorkers.current.clear();
     };
   }, []);
 

src/workers/scrcpyDecoder.ts

@@ -0,0 +1,36 @@
+import { ScrcpyVideoCodecId, type ScrcpyMediaStreamPacket } from "@yume-chan/scrcpy";
+import {
+  WebGLVideoFrameRenderer,
+  BitmapVideoFrameRenderer,
+  WebCodecsVideoDecoder,
+} from "@yume-chan/scrcpy-decoder-webcodecs";
+
+self.addEventListener("message", (e) => {
+  const { codec, canvas, stream, useH265 } = e.data as {
+    codec: ScrcpyVideoCodecId;
+    canvas: OffscreenCanvas;
+    stream: ReadableStream<ScrcpyMediaStreamPacket>;
+    useH265: boolean;
+  };
+
+  let renderer;
+  if (WebGLVideoFrameRenderer.isSupported) {
+    renderer = new WebGLVideoFrameRenderer(canvas);
+  } else {
+    renderer = new BitmapVideoFrameRenderer(canvas);
+  }
+
+  const decoder = new WebCodecsVideoDecoder({
+    codec: codec,
+    renderer: renderer,
+    hardwareAcceleration: useH265 ? "no-preference" : "prefer-software",
+  });
+
+  decoder.sizeChanged(({ width, height }) => {
+    postMessage({ type: 'sizeChanged', width, height });
+  });
+
+  void stream.pipeTo(decoder.writable).catch((err) => {
+    console.error("[Worker] Error piping to decoder writable stream:", err);
+  });
+});

tsconfig.app.json

@@ -2,7 +2,7 @@
   "compilerOptions": {
     "composite": true,
     "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo",
-    "target": "ES2021",
+    "target": "ES2022",
     "useDefineForClassFields": true,
     "lib": ["ES2021", "DOM", "DOM.Iterable"],
     "module": "ESNext",