feat(term): add sixel rendering and propagate terminal pixel size to PTY

frontend/app/view/term/term-model.ts

@@ -738,10 +738,7 @@ export class TermViewModel implements ViewModel {
         }
         this.triggerRestartAtom();
         await RpcApi.ControllerDestroyCommand(TabRpcClient, this.blockId);
-        const termsize = {
-            rows: this.termRef.current?.terminal?.rows,
-            cols: this.termRef.current?.terminal?.cols,
-        };
+        const termsize = this.termRef.current?.getTermSize();
         await RpcApi.ControllerResyncCommand(TabRpcClient, {
             tabid: globalStore.get(atoms.staticTabId),
             blockid: this.blockId,
@@ -756,10 +753,7 @@ export class TermViewModel implements ViewModel {
             meta: { "term:durable": isDurable },
         });
         await RpcApi.ControllerDestroyCommand(TabRpcClient, this.blockId);
-        const termsize = {
-            rows: this.termRef.current?.terminal?.rows,
-            cols: this.termRef.current?.terminal?.cols,
-        };
+        const termsize = this.termRef.current?.getTermSize();
         await RpcApi.ControllerResyncCommand(TabRpcClient, {
             tabid: globalStore.get(atoms.staticTabId),
             blockid: this.blockId,

frontend/app/view/term/termwrap.ts

@@ -23,6 +23,7 @@ import { SearchAddon } from "@xterm/addon-search";
 import { SerializeAddon } from "@xterm/addon-serialize";
 import { WebLinksAddon } from "@xterm/addon-web-links";
 import { WebglAddon } from "@xterm/addon-webgl";
+import { ImageAddon } from "@xterm/addon-image";
 import * as TermTypes from "@xterm/xterm";
 import { Terminal } from "@xterm/xterm";
 import debug from "debug";
@@ -63,6 +64,7 @@ let loggedWebGL = false;
 type TermWrapOptions = {
     keydownHandler?: (e: KeyboardEvent) => boolean;
     useWebGl?: boolean;
+    useSixel?: boolean;
     sendDataHandler?: (data: string) => void;
     nodeModel?: BlockNodeModel;
 };
@@ -193,6 +195,19 @@ export class TermWrap {
                 loggedWebGL = true;
             }
         }
+        if (waveOptions.useSixel ?? true) {
+            try {
+                this.terminal.loadAddon(
+                    new ImageAddon({
+                        enableSizeReports: true,
+                        sixelSupport: true,
+                        iipSupport: false,
+                    })
+                );
+            } catch (e) {
+                console.error("failed to load image addon for sixel support", e);
+            }
+        }
         // Register OSC handlers
         this.terminal.parser.registerOscHandler(7, (data: string) => {
             return handleOsc7Command(data, this.blockId, this.loaded);
@@ -493,6 +508,35 @@ export class TermWrap {
         return prtn;
     }
 
+    private getTerminalPixelSize(): { xpixel: number; ypixel: number } {
+        const screenElem = this.connectElem.querySelector(".xterm-screen") as HTMLElement | null;
+        const targetElem = screenElem ?? this.connectElem;
+        const rect = targetElem.getBoundingClientRect();
+        return {
+            xpixel: Math.max(0, Math.floor(rect.width)),
+            ypixel: Math.max(0, Math.floor(rect.height)),
+        };
+    }
+
+    private areTermSizesEqual(a: TermSize, b: TermSize): boolean {
+        return (
+            a.rows === b.rows &&
+            a.cols === b.cols &&
+            (a.xpixel ?? 0) === (b.xpixel ?? 0) &&
+            (a.ypixel ?? 0) === (b.ypixel ?? 0)
+        );
+    }
+
+    getTermSize(): TermSize {
+        const { xpixel, ypixel } = this.getTerminalPixelSize();
+        const termSize: TermSize = { rows: this.terminal.rows, cols: this.terminal.cols };
+        if (xpixel > 0 && ypixel > 0) {
+            termSize.xpixel = xpixel;
+            termSize.ypixel = ypixel;
+        }
+        return termSize;
+    }
+
     async loadInitialTerminalData(): Promise<void> {
         const startTs = Date.now();
         const zoneId = this.getZoneId();
@@ -501,7 +545,7 @@ export class TermWrap {
         if (cacheFile != null) {
             ptyOffset = cacheFile.meta["ptyoffset"] ?? 0;
             if (cacheData.byteLength > 0) {
-                const curTermSize: TermSize = { rows: this.terminal.rows, cols: this.terminal.cols };
+                const curTermSize: TermSize = this.getTermSize();
                 const fileTermSize: TermSize = cacheFile.meta["termsize"];
                 let didResize = false;
                 if (
@@ -529,7 +573,7 @@ export class TermWrap {
 
     async resyncController(reason: string) {
         dlog("resync controller", this.blockId, reason);
-        const rtOpts: RuntimeOpts = { termsize: { rows: this.terminal.rows, cols: this.terminal.cols } };
+        const rtOpts: RuntimeOpts = { termsize: this.getTermSize() };
         try {
             await RpcApi.ControllerResyncCommand(TabRpcClient, {
                 tabid: this.tabId,
@@ -572,26 +616,30 @@ export class TermWrap {
     }
 
     handleResize() {
-        const oldRows = this.terminal.rows;
-        const oldCols = this.terminal.cols;
+        const oldTermSize = this.getTermSize();
         const atBottom = this.cachedAtBottomForResize ?? this.wasRecentlyAtBottom();
         if (!atBottom) {
             this.cachedAtBottomForResize = null;
         }
         this.fitAddon.fit();
-        if (oldRows !== this.terminal.rows || oldCols !== this.terminal.cols) {
-            const termSize: TermSize = { rows: this.terminal.rows, cols: this.terminal.cols };
+        const newTermSize = this.getTermSize();
+        if (!this.areTermSizesEqual(oldTermSize, newTermSize)) {
             console.log(
                 "[termwrap] resize",
-                `${oldRows}x${oldCols}`,
+                `${oldTermSize.rows}x${oldTermSize.cols}`,
                 "->",
-                `${this.terminal.rows}x${this.terminal.cols}`,
+                `${newTermSize.rows}x${newTermSize.cols}`,
                 "atBottom:",
                 atBottom
             );
-            RpcApi.ControllerInputCommand(TabRpcClient, { blockid: this.blockId, termsize: termSize });
+            RpcApi.ControllerInputCommand(TabRpcClient, { blockid: this.blockId, termsize: newTermSize });
         }
-        dlog("resize", `${this.terminal.rows}x${this.terminal.cols}`, `${oldRows}x${oldCols}`, this.hasResized);
+        dlog(
+            "resize",
+            `${newTermSize.rows}x${newTermSize.cols}`,
+            `${oldTermSize.rows}x${oldTermSize.cols}`,
+            this.hasResized
+        );
         if (!this.hasResized) {
             this.hasResized = true;
             this.resyncController("initial resize");
@@ -611,7 +659,7 @@ export class TermWrap {
             return;
         }
         const serializedOutput = this.serializeAddon.serialize();
-        const termSize: TermSize = { rows: this.terminal.rows, cols: this.terminal.cols };
+        const termSize: TermSize = this.getTermSize();
         console.log("idle timeout term", this.dataBytesProcessed, serializedOutput.length, termSize);
         fireAndForget(() =>
             services.BlockService.SaveTerminalState(this.blockId, serializedOutput, "full", this.ptyOffset, termSize)

frontend/types/gotypes.d.ts

@@ -1586,6 +1586,8 @@ declare global {
     type TermSize = {
         rows: number;
         cols: number;
+        xpixel?: number;
+        ypixel?: number;
     };
 
     // wconfig.TermThemeType