From 85270904e16655137ed435b67f3892875875707c Mon Sep 17 00:00:00 2001 From: cuttlefisch Date: Sun, 5 Apr 2026 12:28:21 +0200 Subject: [PATCH 1/2] Fix xovi checksum validation and harden full extension lifecycle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: QMD files lack .gitattributes protection, so Windows core.autocrlf=true converts LF→CRLF, breaking all SHA-512 checksums. Fixes: - Add .gitattributes to enforce LF line endings for QMD and shell files - Normalize line endings before hashing (defense-in-depth) in manifest generation, validation script, and runtime verification - Add runtime checksum verification before deploying to device - Check vellum reenable status before xovi deploy (was only on install) - Clean orphaned QMD files before vellum removal - Persist rebuild_hashtable warnings in done event (was ephemeral) - Show supported firmware version range in UI when firmware unsupported - Add structured error details and operation names to ErrorDetails - Parse vellum errors into actionable user hints (6 categories) Tests: 13 new integration tests (full xovi lifecycle via mock SSH), 12 unit tests (normalizeLF, sha512Normalized, verifyQmdChecksum), 9 tests for parseVellumErrorHint. All 1027 tests passing. Co-Authored-By: Claude Opus 4.6 --- .gitattributes | 6 + scripts/generate-xovi-manifest.ts | 13 +- scripts/validate-xovi-checksums.ts | 41 ++- server/__tests__/helpers/mockSshServer.ts | 112 +++++++- server/__tests__/helpers/seedDeviceFs.ts | 26 ++ server/__tests__/xoviExtensions.test.ts | 97 +++++++ server/__tests__/xoviIntegration.test.ts | 314 ++++++++++++++++++++++ server/__tests__/xoviRoutes.test.ts | 50 ++++ server/lib/xoviExtensions.ts | 39 +++ server/routes/device/xovi.ts | 126 ++++++++- src/components/device/DeviceXoviCard.tsx | 36 ++- src/components/device/ErrorDetails.tsx | 23 +- src/components/device/deviceOpHelpers.ts | 7 +- src/pages/DevicePage.css | 24 ++ 14 files changed, 888 insertions(+), 26 deletions(-) create mode 100644 .gitattributes create mode 100644 server/__tests__/xoviIntegration.test.ts diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9914124 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# QMD extension files are text-like but checksummed — force LF to prevent +# CRLF conversion on Windows from breaking SHA-512 validation. +*.qmd text eol=lf + +# Keep shell scripts consistent +*.sh text eol=lf diff --git a/scripts/generate-xovi-manifest.ts b/scripts/generate-xovi-manifest.ts index 439ae4a..49fbfe7 100644 --- a/scripts/generate-xovi-manifest.ts +++ b/scripts/generate-xovi-manifest.ts @@ -54,8 +54,19 @@ const EXTENSION_DEFS = { }, } +/** Normalize CRLF → LF so checksums are stable across platforms. */ +function normalizeLF(buf: Buffer): Buffer { + if (!buf.includes(0x0d)) return buf + const out: number[] = [] + for (let i = 0; i < buf.length; i++) { + if (buf[i] === 0x0d && i + 1 < buf.length && buf[i + 1] === 0x0a) continue + out.push(buf[i]!) + } + return Buffer.from(out) +} + function sha512(filePath: string): string { - const content = readFileSync(filePath) + const content = normalizeLF(readFileSync(filePath)) return createHash('sha512').update(content).digest('hex') } diff --git a/scripts/validate-xovi-checksums.ts b/scripts/validate-xovi-checksums.ts index 86158a4..f197499 100644 --- a/scripts/validate-xovi-checksums.ts +++ b/scripts/validate-xovi-checksums.ts @@ -2,6 +2,9 @@ * Validate SHA-512 checksums for bundled xovi extension QMD files. * Exits with code 1 if any checksum mismatches. * + * Normalizes line endings (CRLF → LF) before hashing so that checksums + * are stable across platforms even without .gitattributes protection. + * * Usage: npx tsx scripts/validate-xovi-checksums.ts */ @@ -14,6 +17,24 @@ const __dirname = dirname(fileURLToPath(import.meta.url)) const XOVI_DATA_DIR = resolve(__dirname, '../server/data/xovi-extensions') const manifestPath = resolve(XOVI_DATA_DIR, 'manifest.json') +/** Normalize CRLF → LF so checksums are stable across platforms. */ +function normalizeLF(buf: Buffer): Buffer { + // Fast path: no CR bytes at all + if (!buf.includes(0x0d)) return buf + // Strip \r from \r\n sequences + const out: number[] = [] + for (let i = 0; i < buf.length; i++) { + if (buf[i] === 0x0d && i + 1 < buf.length && buf[i + 1] === 0x0a) continue + out.push(buf[i]!) + } + return Buffer.from(out) +} + +/** Compute SHA-512 of a buffer after LF normalization. */ +export function sha512Normalized(buf: Buffer): string { + return createHash('sha512').update(normalizeLF(buf)).digest('hex') +} + if (!existsSync(manifestPath)) { console.error('manifest.json not found — run generate-xovi-manifest.ts first') process.exit(1) @@ -25,6 +46,7 @@ const manifest = JSON.parse(readFileSync(manifestPath, 'utf-8')) as { let ok = true let checked = 0 +let crlfDetected = false for (const [relPath, expectedHash] of Object.entries(manifest.checksums)) { const filePath = resolve(XOVI_DATA_DIR, relPath) @@ -33,8 +55,9 @@ for (const [relPath, expectedHash] of Object.entries(manifest.checksums)) { ok = false continue } - const content = readFileSync(filePath) - const actual = createHash('sha512').update(content).digest('hex') + const raw = readFileSync(filePath) + if (raw.includes(0x0d)) crlfDetected = true + const actual = sha512Normalized(raw) if (actual !== expectedHash) { console.error(`MISMATCH ${relPath}`) console.error(` expected: ${expectedHash}`) @@ -47,7 +70,21 @@ for (const [relPath, expectedHash] of Object.entries(manifest.checksums)) { if (ok) { console.log(`All ${checked} QMD file checksums verified.`) + if (crlfDetected) { + console.warn( + 'WARNING: Some QMD files contain CRLF line endings (normalized before hashing).\n' + + 'Run "git add --renormalize ." to fix, or ensure .gitattributes is present.', + ) + } } else { console.error('\nChecksum validation failed.') + if (crlfDetected) { + console.error( + 'HINT: CRLF line endings detected — this is a common cause of checksum mismatches.\n' + + 'Ensure .gitattributes marks *.qmd with eol=lf, then re-checkout:\n' + + ' git rm --cached server/data/xovi-extensions/**/*.qmd\n' + + ' git checkout -- server/data/xovi-extensions/', + ) + } process.exit(1) } diff --git a/server/__tests__/helpers/mockSshServer.ts b/server/__tests__/helpers/mockSshServer.ts index f0f96d4..fa7bdd0 100644 --- a/server/__tests__/helpers/mockSshServer.ts +++ b/server/__tests__/helpers/mockSshServer.ts @@ -176,9 +176,13 @@ function handleExec( return } - if (command.includes('mkdir -p') && command.includes('.ssh')) { - const sshDir = mapPath(fsRoot, '/home/root/.ssh') - mkdirSync(sshDir, { recursive: true }) + if (command.includes('mkdir -p')) { + // Extract the path argument + const mkdirMatch = command.match(/mkdir -p ([^\s;]+)/) + if (mkdirMatch) { + const dir = mapPath(fsRoot, mkdirMatch[1]) + mkdirSync(dir, { recursive: true }) + } channel.exit(0) channel.end() return @@ -227,6 +231,108 @@ function handleExec( return } + // Generic `test -f && echo ok || echo missing` pattern + const testFileMatch = command.match(/test -f ([^\s&|]+)\s*&&\s*echo ok\s*\|\|\s*echo missing/) + if (testFileMatch) { + const localPath = mapPath(fsRoot, testFileMatch[1]) + channel.write(existsSync(localPath) ? 'ok' : 'missing') + channel.exit(0) + channel.end() + return + } + + // Generic `test -x && echo ok || echo missing` pattern + const testExecMatch = command.match(/test -x ([^\s&|]+)\s*&&\s*echo ok\s*\|\|\s*echo missing/) + if (testExecMatch) { + const localPath = mapPath(fsRoot, testExecMatch[1]) + channel.write(existsSync(localPath) ? 'ok' : 'missing') + channel.exit(0) + channel.end() + return + } + + // Generic `test -f && || echo missing` — for vellum version check etc. + const testAndExecMatch = command.match(/test -f ([^\s&|]+)\s*&&/) + if (testAndExecMatch && !command.includes('echo ok')) { + const localPath = mapPath(fsRoot, testAndExecMatch[1]) + if (!existsSync(localPath)) { + channel.write('missing') + channel.exit(1) + channel.end() + return + } + // File exists — fall through to more specific handlers below + } + + // Vellum reenable status + if (command.includes('vellum') && command.includes('reenable status')) { + const marker = mapPath(fsRoot, '/home/root/.vellum/.reenable-needed') + channel.write(existsSync(marker) ? 'needed' : 'ok') + channel.exit(0) + channel.end() + return + } + + // Vellum --version + if (command.includes('vellum') && command.includes('--version')) { + const vellumBin = mapPath(fsRoot, '/home/root/.vellum/bin/vellum') + channel.write(existsSync(vellumBin) ? '1.0.0' : 'missing') + channel.exit(existsSync(vellumBin) ? 0 : 1) + channel.end() + return + } + + // Vellum add/del — always succeed (mock) + if (command.includes('vellum') && (command.includes(' add ') || command.includes(' del '))) { + channel.write('ok\n') + channel.exit(0) + channel.end() + return + } + + // rebuild_hashtable + if (command.includes('rebuild_hashtable')) { + const bin = mapPath(fsRoot, '/home/root/xovi/rebuild_hashtable') + if (existsSync(bin)) { + channel.write('Hashtable rebuilt successfully\n') + channel.exit(0) + } else { + channel.stderr.write('rebuild_hashtable: not found\n') + channel.exit(1) + } + channel.end() + return + } + + // rm -f — delete a file (best effort) + const rmMatch = command.match(/rm -f ([^\s;]+)/) + if (rmMatch) { + const localPath = mapPath(fsRoot, rmMatch[1]) + if (existsSync(localPath)) { + try { unlinkSync(localPath) } catch { /* best effort */ } + } + channel.exit(0) + channel.end() + return + } + + // rm -f pattern via shell (e.g. rm -f /path/*.qmd) + if (command.includes('rm -f') && command.includes('*.qmd')) { + // Extract dir from the glob pattern + const globMatch = command.match(/rm -f ([^\s*]+)\*\.qmd/) + if (globMatch) { + const dir = mapPath(fsRoot, globMatch[1]) + if (existsSync(dir)) { + for (const f of readdirSync(dir)) { + if (f.endsWith('.qmd')) { + try { unlinkSync(resolve(dir, f)) } catch { /* best effort */ } + } + } + } + } + // Fall through to default (will echo ok if piped) + } + // Default: succeed silently channel.exit(0) channel.end() diff --git a/server/__tests__/helpers/seedDeviceFs.ts b/server/__tests__/helpers/seedDeviceFs.ts index 600035c..65488f1 100644 --- a/server/__tests__/helpers/seedDeviceFs.ts +++ b/server/__tests__/helpers/seedDeviceFs.ts @@ -47,6 +47,32 @@ export function seedMethodsTemplates( ) } +/** Seed xovi/vellum filesystem structure for xovi integration tests. */ +export function seedXoviFs( + fsRoot: string, + opts?: { xovi?: boolean; qtRebuilder?: boolean; vellum?: boolean; reenableNeeded?: boolean }, +) { + const { xovi = true, qtRebuilder = true, vellum = true, reenableNeeded = false } = opts ?? {} + if (xovi) { + mkdirSync(resolve(fsRoot, 'home/root/xovi'), { recursive: true }) + writeFileSync(resolve(fsRoot, 'home/root/xovi/xovi.so'), 'fake-xovi') + writeFileSync(resolve(fsRoot, 'home/root/xovi/rebuild_hashtable'), '#!/bin/sh\necho ok') + } + if (qtRebuilder) { + mkdirSync(resolve(fsRoot, 'home/root/xovi/extensions.d'), { recursive: true }) + writeFileSync(resolve(fsRoot, 'home/root/xovi/extensions.d/qt-resource-rebuilder.so'), 'fake-qt') + mkdirSync(resolve(fsRoot, 'home/root/xovi/exthome/qt-resource-rebuilder'), { recursive: true }) + } + if (vellum) { + mkdirSync(resolve(fsRoot, 'home/root/.vellum/bin'), { recursive: true }) + writeFileSync(resolve(fsRoot, 'home/root/.vellum/bin/vellum'), '#!/bin/sh\necho 1.0.0') + } + // Marker file read by mock exec handler + if (reenableNeeded) { + writeFileSync(resolve(fsRoot, 'home/root/.vellum/.reenable-needed'), '') + } +} + /** Write classic templates (templates.json + .template files) to /usr/share/remarkable/templates/. */ export function seedClassicTemplates( fsRoot: string, diff --git a/server/__tests__/xoviExtensions.test.ts b/server/__tests__/xoviExtensions.test.ts index 8efdc4b..0613da6 100644 --- a/server/__tests__/xoviExtensions.test.ts +++ b/server/__tests__/xoviExtensions.test.ts @@ -1,11 +1,17 @@ // @vitest-environment node import { describe, it, expect, beforeEach } from 'vitest' +import { readFileSync, writeFileSync, unlinkSync } from 'node:fs' +import { resolve } from 'node:path' +import { tmpdir } from 'node:os' import { mapFirmwareToQmdVersion, getExtensionDefs, getQmdFilePath, validateExclusiveGroups, getSupportedVersions, + normalizeLF, + sha512Normalized, + verifyQmdChecksum, _resetManifestCache, } from '../lib/xoviExtensions.ts' @@ -143,3 +149,94 @@ describe('getSupportedVersions', () => { } }) }) + +// ── checksum utilities ────────────────────────────────────────────────────── + +describe('normalizeLF', () => { + it('returns the same buffer when no CR bytes present', () => { + const buf = Buffer.from('hello\nworld\n') + const result = normalizeLF(buf) + expect(result).toEqual(buf) + }) + + it('strips CR from CRLF sequences', () => { + const buf = Buffer.from('hello\r\nworld\r\n') + const result = normalizeLF(buf) + expect(result).toEqual(Buffer.from('hello\nworld\n')) + }) + + it('preserves bare CR that is not followed by LF', () => { + const buf = Buffer.from('hello\rworld\n') + const result = normalizeLF(buf) + expect(result).toEqual(Buffer.from('hello\rworld\n')) + }) + + it('handles mixed line endings', () => { + const buf = Buffer.from('a\r\nb\nc\r\nd\r') + const result = normalizeLF(buf) + expect(result).toEqual(Buffer.from('a\nb\nc\nd\r')) + }) + + it('handles empty buffer', () => { + const result = normalizeLF(Buffer.alloc(0)) + expect(result.length).toBe(0) + }) +}) + +describe('sha512Normalized', () => { + it('produces same hash for LF and CRLF versions of the same content', () => { + const lf = Buffer.from('line1\nline2\nline3\n') + const crlf = Buffer.from('line1\r\nline2\r\nline3\r\n') + expect(sha512Normalized(lf)).toBe(sha512Normalized(crlf)) + }) + + it('produces a 128-character hex string', () => { + const hash = sha512Normalized(Buffer.from('test')) + expect(hash).toMatch(/^[0-9a-f]{128}$/) + }) +}) + +describe('verifyQmdChecksum', () => { + it('verifies a known good QMD file', () => { + const result = verifyQmdChecksum('unlockMethodsContent', '3.26', + getQmdFilePath('unlockMethodsContent', '3.26')) + expect(result.ok).toBe(true) + expect(result.expected).toBeTruthy() + expect(result.actual).toBe(result.expected) + }) + + it('verifies all bundled QMD files pass checksum', () => { + for (const version of getSupportedVersions()) { + for (const def of getExtensionDefs()) { + let filePath: string + try { + filePath = getQmdFilePath(def.id, version) + } catch { + continue // extension not available in this version + } + const result = verifyQmdChecksum(def.id, version, filePath) + expect(result.ok, `${def.id}@${version}`).toBe(true) + } + } + }) + + it('detects a tampered file', () => { + const filePath = getQmdFilePath('unlockMethodsContent', '3.26') + const content = readFileSync(filePath) + const tampered = Buffer.concat([content, Buffer.from('X')]) + const tmpPath = resolve(tmpdir(), `tampered-${Date.now()}.qmd`) + writeFileSync(tmpPath, tampered) + try { + const result = verifyQmdChecksum('unlockMethodsContent', '3.26', tmpPath) + expect(result.ok).toBe(false) + expect(result.actual).not.toBe(result.expected) + } finally { + unlinkSync(tmpPath) + } + }) + + it('throws for unknown extension', () => { + expect(() => verifyQmdChecksum('nonexistent', '3.26', '/tmp/fake.qmd')) + .toThrow('Unknown extension') + }) +}) diff --git a/server/__tests__/xoviIntegration.test.ts b/server/__tests__/xoviIntegration.test.ts new file mode 100644 index 0000000..78943fc --- /dev/null +++ b/server/__tests__/xoviIntegration.test.ts @@ -0,0 +1,314 @@ +// @vitest-environment node +/** + * SSH integration tests for xovi extension routes. + * + * Uses an in-process ssh2 mock server backed by a real temp directory. + * Tests the full xovi lifecycle: status, deploy, remove, vellum install/remove. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from 'vitest' +import { mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { tmpdir } from 'node:os' +import { createApp } from '../app.ts' +import { resolveConfig, type ServerConfig } from '../config.ts' +import { writeDeviceStore } from '../lib/deviceStore.ts' +import type { DeviceConfig } from '../lib/ssh.ts' +import { startMockSshServer, type MockSshServer } from './helpers/mockSshServer.ts' +import { seedXoviFs } from './helpers/seedDeviceFs.ts' +import { parseNdjson } from './helpers/ndjsonHelper.ts' + +const TEST_TIMEOUT = 15_000 + +function makeConfig(): ServerConfig { + const base = resolve(tmpdir(), `xovi-integ-${Date.now()}-${Math.random().toString(36).slice(2)}`) + mkdirSync(resolve(base, 'public/templates/custom'), { recursive: true }) + mkdirSync(resolve(base, 'public/templates/debug'), { recursive: true }) + mkdirSync(resolve(base, 'public/templates/methods'), { recursive: true }) + mkdirSync(resolve(base, 'public/templates/samples'), { recursive: true }) + mkdirSync(resolve(base, 'remarkable_official_templates'), { recursive: true }) + mkdirSync(resolve(base, 'rm-methods-dist'), { recursive: true }) + mkdirSync(resolve(base, 'rm-methods-backups'), { recursive: true }) + mkdirSync(resolve(base, 'data/ssh'), { recursive: true }) + return resolveConfig({ dataDir: base, port: 0, production: false }) +} + +function seedDevice( + config: ServerConfig, + mockServer: MockSshServer, + opts?: { firmwareVersion?: string }, +): DeviceConfig { + const device: DeviceConfig = { + id: 'test-dev-1', + nickname: 'Test RM', + deviceIp: '127.0.0.1', + sshPort: mockServer.port, + authMethod: 'password', + sshPassword: 'test', + firmwareVersion: opts?.firmwareVersion ?? '3.26.1.2', + deviceModel: 'rmPP', + } + writeDeviceStore(config.deviceConfigPath, { + version: 2, + devices: [device], + activeDeviceId: device.id, + }) + return device +} + +describe('xovi SSH integration', () => { + let mockServer: MockSshServer + let config: ServerConfig + + beforeAll(async () => { + mockServer = await startMockSshServer() + }) + + afterAll(async () => { + await mockServer.close() + }) + + beforeEach(() => { + config = makeConfig() + mockServer.resetFs() + }) + + afterEach(() => { + rmSync(config.dataDir, { recursive: true, force: true }) + }) + + // ── xovi-status ────────────────────────────────────────────────────────── + + describe('POST /api/devices/:id/xovi-status', () => { + it('returns full status with xovi installed', async () => { + seedDevice(config, mockServer) + seedXoviFs(mockServer.fsRoot) + const app = await createApp(config) + const res = await app.inject({ method: 'POST', url: '/api/devices/test-dev-1/xovi-status' }) + expect(res.statusCode).toBe(200) + const body = JSON.parse(res.body) + expect(body.ok).toBe(true) + expect(body.xoviInstalled).toBe(true) + expect(body.qtRebuilderInstalled).toBe(true) + expect(body.vellumInstalled).toBe(true) + expect(body.qmdVersion).toBe('3.26') + expect(body.supportedVersionRange).toEqual({ min: '3.22', max: '3.26' }) + await app.close() + }, TEST_TIMEOUT) + + it('reports unsupported firmware when version is too new', async () => { + seedDevice(config, mockServer, { firmwareVersion: '3.99.0.0' }) + seedXoviFs(mockServer.fsRoot) + const app = await createApp(config) + const res = await app.inject({ method: 'POST', url: '/api/devices/test-dev-1/xovi-status' }) + expect(res.statusCode).toBe(200) + const body = JSON.parse(res.body) + expect(body.qmdVersion).toBeNull() + expect(body.supportedVersionRange).toEqual({ min: '3.22', max: '3.26' }) + await app.close() + }, TEST_TIMEOUT) + + it('detects vellum reenable needed', async () => { + seedDevice(config, mockServer) + seedXoviFs(mockServer.fsRoot, { reenableNeeded: true }) + const app = await createApp(config) + const res = await app.inject({ method: 'POST', url: '/api/devices/test-dev-1/xovi-status' }) + expect(res.statusCode).toBe(200) + const body = JSON.parse(res.body) + expect(body.vellumReenableNeeded).toBe(true) + await app.close() + }, TEST_TIMEOUT) + }) + + // ── xovi-deploy ────────────────────────────────────────────────────────── + + describe('POST /api/devices/:id/xovi-deploy', () => { + it('deploys extensions successfully', async () => { + seedDevice(config, mockServer) + seedXoviFs(mockServer.fsRoot) + const app = await createApp(config) + const res = await app.inject({ + method: 'POST', + url: '/api/devices/test-dev-1/xovi-deploy', + payload: { extensionIds: ['unlockMethodsContent'] }, + }) + expect(res.statusCode).toBe(200) + const events = parseNdjson(res.body) + const done = events.find(e => e.type === 'done') + expect(done).toBeTruthy() + expect((done as Record).extensions).toEqual(['unlockMethodsContent']) + await app.close() + }, TEST_TIMEOUT) + + it('blocks deploy when vellum reenable is needed', async () => { + seedDevice(config, mockServer) + seedXoviFs(mockServer.fsRoot, { reenableNeeded: true }) + const app = await createApp(config) + const res = await app.inject({ + method: 'POST', + url: '/api/devices/test-dev-1/xovi-deploy', + payload: { extensionIds: ['unlockMethodsContent'] }, + }) + expect(res.statusCode).toBe(200) + const events = parseNdjson(res.body) + const error = events.find(e => e.type === 'error') + expect(error).toBeTruthy() + expect((error as Record).error).toMatch(/firmware was updated/) + expect((error as Record).hint).toMatch(/vellum reenable/) + await app.close() + }, TEST_TIMEOUT) + + it('blocks deploy when xovi is not installed', async () => { + seedDevice(config, mockServer) + // No seedXoviFs — xovi not installed + const app = await createApp(config) + const res = await app.inject({ + method: 'POST', + url: '/api/devices/test-dev-1/xovi-deploy', + payload: { extensionIds: ['unlockMethodsContent'] }, + }) + expect(res.statusCode).toBe(200) + const events = parseNdjson(res.body) + const error = events.find(e => e.type === 'error') + expect(error).toBeTruthy() + expect((error as Record).error).toMatch(/not installed/) + await app.close() + }, TEST_TIMEOUT) + + it('returns 400 for unsupported firmware version with version range in hint', async () => { + seedDevice(config, mockServer, { firmwareVersion: '3.99.0.0' }) + seedXoviFs(mockServer.fsRoot) + const app = await createApp(config) + const res = await app.inject({ + method: 'POST', + url: '/api/devices/test-dev-1/xovi-deploy', + payload: { extensionIds: ['unlockMethodsContent'] }, + }) + expect(res.statusCode).toBe(400) + const body = JSON.parse(res.body) + expect(body.error).toMatch(/No extensions available/) + expect(body.hint).toMatch(/3\.22/) + expect(body.hint).toMatch(/3\.26/) + expect(body.hint).toMatch(/firmware-specific/) + await app.close() + }, TEST_TIMEOUT) + }) + + // ── xovi-remove ────────────────────────────────────────────────────────── + + describe('POST /api/devices/:id/xovi-remove', () => { + it('removes extensions successfully', async () => { + seedDevice(config, mockServer) + seedXoviFs(mockServer.fsRoot) + // Put a fake QMD file on "device" + const qmdDir = resolve(mockServer.fsRoot, 'home/root/xovi/exthome/qt-resource-rebuilder') + writeFileSync(resolve(qmdDir, 'unlockMethodsContent.qmd'), 'fake-qmd') + const app = await createApp(config) + const res = await app.inject({ + method: 'POST', + url: '/api/devices/test-dev-1/xovi-remove', + payload: { extensionIds: ['unlockMethodsContent'] }, + }) + expect(res.statusCode).toBe(200) + const events = parseNdjson(res.body) + const done = events.find(e => e.type === 'done') + expect(done).toBeTruthy() + expect((done as Record).removed).toEqual(['unlockMethodsContent']) + await app.close() + }, TEST_TIMEOUT) + }) + + // ── vellum-install-xovi ────────────────────────────────────────────────── + + describe('POST /api/devices/:id/vellum-install-xovi', () => { + it('installs xovi when vellum is ready', async () => { + seedDevice(config, mockServer) + seedXoviFs(mockServer.fsRoot, { xovi: false, qtRebuilder: false, vellum: true }) + const app = await createApp(config) + const res = await app.inject({ + method: 'POST', + url: '/api/devices/test-dev-1/vellum-install-xovi', + }) + expect(res.statusCode).toBe(200) + const events = parseNdjson(res.body) + const done = events.find(e => e.type === 'done') + expect(done).toBeTruthy() + expect((done as Record).message).toMatch(/installed successfully/) + await app.close() + }, TEST_TIMEOUT) + + it('blocks install when vellum reenable is needed', async () => { + seedDevice(config, mockServer) + seedXoviFs(mockServer.fsRoot, { vellum: true, reenableNeeded: true }) + const app = await createApp(config) + const res = await app.inject({ + method: 'POST', + url: '/api/devices/test-dev-1/vellum-install-xovi', + }) + expect(res.statusCode).toBe(200) + const events = parseNdjson(res.body) + const error = events.find(e => e.type === 'error') + expect(error).toBeTruthy() + expect((error as Record).error).toMatch(/re-enabled/) + await app.close() + }, TEST_TIMEOUT) + + it('errors when vellum is not installed', async () => { + seedDevice(config, mockServer) + // No vellum + const app = await createApp(config) + const res = await app.inject({ + method: 'POST', + url: '/api/devices/test-dev-1/vellum-install-xovi', + }) + expect(res.statusCode).toBe(200) + const events = parseNdjson(res.body) + const error = events.find(e => e.type === 'error') + expect(error).toBeTruthy() + expect((error as Record).error).toMatch(/not installed/) + await app.close() + }, TEST_TIMEOUT) + }) + + // ── vellum-remove-xovi ─────────────────────────────────────────────────── + + describe('POST /api/devices/:id/vellum-remove-xovi', () => { + it('cleans QMD files before uninstalling xovi', async () => { + seedDevice(config, mockServer) + seedXoviFs(mockServer.fsRoot) + // Put QMD files on "device" + const qmdDir = resolve(mockServer.fsRoot, 'home/root/xovi/exthome/qt-resource-rebuilder') + writeFileSync(resolve(qmdDir, 'unlockMethodsContent.qmd'), 'fake-qmd') + const app = await createApp(config) + const res = await app.inject({ + method: 'POST', + url: '/api/devices/test-dev-1/vellum-remove-xovi', + }) + expect(res.statusCode).toBe(200) + const events = parseNdjson(res.body) + const done = events.find(e => e.type === 'done') + expect(done).toBeTruthy() + expect((done as Record).message).toMatch(/removed successfully/) + // Verify steps include expected operations + const steps = (done as Record).steps as string[] + expect(steps).toBeDefined() + expect(steps.some(s => s.includes('Removed xovi') || s.includes('Cleaned QMD'))).toBe(true) + await app.close() + }, TEST_TIMEOUT) + + it('errors when vellum is not installed', async () => { + seedDevice(config, mockServer) + // No vellum + const app = await createApp(config) + const res = await app.inject({ + method: 'POST', + url: '/api/devices/test-dev-1/vellum-remove-xovi', + }) + expect(res.statusCode).toBe(200) + const events = parseNdjson(res.body) + const error = events.find(e => e.type === 'error') + expect(error).toBeTruthy() + expect((error as Record).error).toMatch(/not installed/) + await app.close() + }, TEST_TIMEOUT) + }) +}) diff --git a/server/__tests__/xoviRoutes.test.ts b/server/__tests__/xoviRoutes.test.ts index 6fdfcd0..8183bc6 100644 --- a/server/__tests__/xoviRoutes.test.ts +++ b/server/__tests__/xoviRoutes.test.ts @@ -6,6 +6,7 @@ import { tmpdir } from 'node:os' import { createApp } from '../app.ts' import { resolveConfig, type ServerConfig } from '../config.ts' import { writeDeviceStore } from '../lib/deviceStore.ts' +import { parseVellumErrorHint } from '../routes/device/xovi.ts' function makeConfig(): ServerConfig { const base = resolve(tmpdir(), `xoviroutes-test-${Date.now()}-${Math.random().toString(36).slice(2)}`) @@ -237,3 +238,52 @@ describe('xovi routes', () => { }) }) }) + +// ── parseVellumErrorHint ────────────────────────────────────────────────── + +describe('parseVellumErrorHint', () => { + it('detects checksum errors', () => { + const hint = parseVellumErrorHint('Error: checksum mismatch for package xovi') + expect(hint).toMatch(/checksum/) + expect(hint).toMatch(/vellum update/) + }) + + it('detects hash mismatch errors', () => { + const hint = parseVellumErrorHint('hash mismatch: expected abc123, got def456') + expect(hint).toMatch(/checksum/) + }) + + it('detects integrity errors', () => { + const hint = parseVellumErrorHint('integrity verification failed') + expect(hint).toMatch(/checksum/) + }) + + it('detects reenable-needed errors', () => { + const hint = parseVellumErrorHint('Please reenable vellum after firmware update') + expect(hint).toMatch(/re-enabled/) + expect(hint).toMatch(/vellum reenable/) + }) + + it('detects network errors', () => { + expect(parseVellumErrorHint('could not resolve host packages.vellum.dev')).toMatch(/internet/) + expect(parseVellumErrorHint('connection refused')).toMatch(/internet/) + expect(parseVellumErrorHint('network unreachable')).toMatch(/internet/) + expect(parseVellumErrorHint('request timeout')).toMatch(/internet/) + }) + + it('detects missing package errors', () => { + const hint = parseVellumErrorHint('Error: no such package xovi-extensions') + expect(hint).toMatch(/not found/) + }) + + it('detects disk space errors', () => { + expect(parseVellumErrorHint('ENOSPC: no space left on device')).toMatch(/disk space/) + expect(parseVellumErrorHint('No space left on device')).toMatch(/disk space/) + }) + + it('returns generic hint for unknown errors', () => { + const hint = parseVellumErrorHint('some unexpected error output') + expect(hint).toMatch(/internet access/) + expect(hint).toMatch(/manually/) + }) +}) diff --git a/server/lib/xoviExtensions.ts b/server/lib/xoviExtensions.ts index 1ec891f..297f839 100644 --- a/server/lib/xoviExtensions.ts +++ b/server/lib/xoviExtensions.ts @@ -5,6 +5,7 @@ * Pure logic (except checkXoviStatus which uses SSH/SFTP). */ +import { createHash } from 'node:crypto' import { readFileSync, existsSync } from 'node:fs' import { resolve, dirname } from 'node:path' import { fileURLToPath } from 'node:url' @@ -77,6 +78,44 @@ export function _resetManifestCache(): void { _manifest = null } +// ── checksum utilities ────────────────────────────────────────────────────── + +/** Normalize CRLF → LF so checksums are stable across platforms. */ +export function normalizeLF(buf: Buffer): Buffer { + if (!buf.includes(0x0d)) return buf + const out: number[] = [] + for (let i = 0; i < buf.length; i++) { + if (buf[i] === 0x0d && i + 1 < buf.length && buf[i + 1] === 0x0a) continue + out.push(buf[i]!) + } + return Buffer.from(out) +} + +/** Compute SHA-512 of a buffer after LF normalization. */ +export function sha512Normalized(buf: Buffer): string { + return createHash('sha512').update(normalizeLF(buf)).digest('hex') +} + +/** + * Verify a QMD file's integrity against the manifest checksum. + * Returns `{ ok, expected, actual }`. If the manifest has no entry + * for the given key, `expected` is null (no checksum to verify against). + */ +export function verifyQmdChecksum( + extensionId: string, + qmdVersion: string, + filePath: string, +): { ok: boolean; expected: string | null; actual: string } { + const manifest = loadManifest() + const def = manifest.extensions[extensionId] + if (!def) throw new Error(`Unknown extension: ${extensionId}`) + const key = `${qmdVersion}/${def.filename}` + const expected = manifest.checksums[key] ?? null + const content = readFileSync(filePath) + const actual = sha512Normalized(content) + return { ok: expected === null || actual === expected, expected, actual } +} + // ── public API ─────────────────────────────────────────────────────────────── /** diff --git a/server/routes/device/xovi.ts b/server/routes/device/xovi.ts index d4de4ff..0905e44 100644 --- a/server/routes/device/xovi.ts +++ b/server/routes/device/xovi.ts @@ -19,11 +19,50 @@ import { checkXoviStatus, getExtensionDefs, getQmdFilePath, + getSupportedVersions, mapFirmwareToQmdVersion, validateExclusiveGroups, + verifyQmdChecksum, DEVICE_PATHS, } from '../../lib/xoviExtensions.ts' +/** + * Parse Vellum command output for known error patterns and return a targeted hint. + */ +export function parseVellumErrorHint(output: string): string { + const lower = output.toLowerCase() + + if (lower.includes('checksum') || lower.includes('hash mismatch') || lower.includes('integrity')) { + return 'Package checksum verification failed. This usually means the package repository is out of sync. ' + + 'Try: SSH into the device and run "vellum update" to refresh package metadata, then retry.' + } + + if (lower.includes('reenable') || lower.includes('re-enable')) { + return 'Vellum needs to be re-enabled after a firmware update. ' + + 'SSH into your device and run: vellum reenable' + } + + if (lower.includes('no route') || lower.includes('could not resolve') || lower.includes('network') + || lower.includes('connection refused') || lower.includes('timeout')) { + return 'The device cannot reach the package server. ' + + 'Check that the device has a working internet connection (Settings → Wi-Fi), then retry.' + } + + if (lower.includes('not found') || lower.includes('no such package')) { + return 'One or more packages were not found in the Vellum repository. ' + + 'The package names may have changed. Check https://github.com/vellum-dev/vellum for updates.' + } + + if (lower.includes('disk') || lower.includes('no space') || lower.includes('enospc')) { + return 'The device is running low on disk space. ' + + 'Free up space by removing unused notebooks or files, then retry.' + } + + return 'Check that the device has internet access and try again. ' + + 'If the problem persists, SSH into the device and run the command manually: ' + + 'vellum add xovi xovi-extensions qt-resource-rebuilder' +} + export default function deviceXoviRoutes(app: FastifyInstance, _config: ServerConfig) { // ── POST /api/devices/:id/xovi-status ────────────────────────────────────── app.post<{ Params: { id: string } }>('/api/devices/:id/xovi-status', async (request, reply) => { @@ -38,7 +77,14 @@ export default function deviceXoviRoutes(app: FastifyInstance, _config: ServerCo client = await connect(deviceConfig) const sftp = await getSftp(client) const status = await checkXoviStatus(client, sftp, deviceConfig.firmwareVersion ?? null) - return reply.send({ ok: true, ...status }) + const supported = getSupportedVersions() + return reply.send({ + ok: true, + ...status, + supportedVersionRange: supported.length > 0 + ? { min: supported[0], max: supported[supported.length - 1] } + : null, + }) } catch (err) { const friendly = formatSshError(err as Error) return reply.status(500).send({ error: friendly.message, hint: friendly.hint, rawError: friendly.rawError }) @@ -84,9 +130,13 @@ export default function deviceXoviRoutes(app: FastifyInstance, _config: ServerCo } const qmdVersion = mapFirmwareToQmdVersion(fw) if (!qmdVersion) { + const supported = getSupportedVersions() + const range = supported.length > 0 ? `${supported[0]}–${supported[supported.length - 1]}` : 'none' return reply.status(400).send({ error: `No extensions available for firmware ${fw}`, - hint: 'Extensions may not yet support this firmware version.', + hint: `This app bundles extensions for firmware ${range}. ` + + 'QMD extensions are firmware-specific and cannot be safely used across versions. ' + + 'Check for an app update, or visit https://github.com/rmitchellscott/xovi for the latest extensions.', }) } @@ -105,6 +155,29 @@ export default function deviceXoviRoutes(app: FastifyInstance, _config: ServerCo } } + // Verify QMD file integrity before connecting to device + const checksumFailures: string[] = [] + for (const { extensionId, localPath } of filePaths) { + try { + const result = verifyQmdChecksum(extensionId, qmdVersion, localPath) + if (!result.ok) { + checksumFailures.push( + `${extensionId}: expected ${result.expected?.slice(0, 16)}..., got ${result.actual.slice(0, 16)}...`, + ) + } + } catch { + checksumFailures.push(`${extensionId}: unable to verify checksum`) + } + } + if (checksumFailures.length > 0) { + return reply.status(400).send({ + error: 'QMD file integrity check failed', + hint: 'Extension files may be corrupted (CRLF line ending conversion is a common cause). ' + + 'Re-clone the repository or run: git rm --cached server/data/xovi-extensions/**/*.qmd && git checkout -- server/data/xovi-extensions/', + details: checksumFailures, + }) + } + const stream = createNdjsonStream(reply) let client: Awaited> | null = null @@ -136,6 +209,21 @@ export default function deviceXoviRoutes(app: FastifyInstance, _config: ServerCo ) return } + // Check vellum reenable status — after a firmware update, QMD patches + // target stale symbol IDs and must not be deployed until the user re-enables + // vellum and installs QMDs matching the new firmware. + const vellumExists = await exec(client, `test -f ${DEVICE_PATHS.vellumBin} && echo ok || echo missing`) + if (vellumExists.stdout.trim() === 'ok') { + const reenableResult = await exec(client, `${DEVICE_PATHS.vellumBin} reenable status 2>/dev/null`) + if (reenableResult.stdout.trim() === 'needed') { + stream.error( + 'Deploy blocked: firmware was updated since xovi was installed', + 'QMD extensions target firmware-specific symbols and cannot be safely deployed until Vellum is re-enabled. ' + + 'SSH into your device and run: vellum reenable — then check xovi status again.', + ) + return + } + } steps.push('xovi prerequisites verified') // Ensure QMD directory exists @@ -229,7 +317,7 @@ export default function deviceXoviRoutes(app: FastifyInstance, _config: ServerCo if (rebuildResult.code !== 0) { stream.error( 'rebuild_hashtable failed after removal', - `Exit code ${rebuildResult.code}`, + `Exit code ${rebuildResult.code}. Try running it manually: ${DEVICE_PATHS.rebuildCmd}`, rebuildResult.stderr, ) return @@ -296,10 +384,11 @@ export default function deviceXoviRoutes(app: FastifyInstance, _config: ServerCo stream.progress('Installing xovi via Vellum (downloading packages)...') const installResult = await exec(client, `${DEVICE_PATHS.vellumBin} add xovi xovi-extensions qt-resource-rebuilder 2>&1`) if (installResult.code !== 0) { + const output = installResult.stdout + installResult.stderr stream.error( 'Failed to install xovi via Vellum', - 'Check that the device has internet access and try again.', - installResult.stdout + installResult.stderr, + parseVellumErrorHint(output), + output, ) return } @@ -342,14 +431,33 @@ export default function deviceXoviRoutes(app: FastifyInstance, _config: ServerCo return } + // Clean up deployed QMD files BEFORE vellum del — after removal, + // rebuild_hashtable won't be available so we must rebuild now. + const warnings: string[] = [] + stream.progress('Removing deployed QMD extensions...') + const qmdClean = await exec(client, `rm -f ${DEVICE_PATHS.qmdDir}/*.qmd 2>/dev/null; echo ok`) + if (qmdClean.stdout.trim() === 'ok') { + const hasRebuilder = await exec(client, `test -x /home/root/xovi/rebuild_hashtable && echo ok || echo missing`) + if (hasRebuilder.stdout.trim() === 'ok') { + stream.progress('Rebuilding hashtable before uninstall...') + const rebuildResult = await exec(client, DEVICE_PATHS.rebuildCmd) + if (rebuildResult.code !== 0) { + warnings.push(`rebuild_hashtable returned exit code ${rebuildResult.code} during cleanup (continuing with uninstall)`) + } else { + steps.push('Cleaned QMD extensions and rebuilt hashtable') + } + } + } + // Remove all three packages explicitly — vellum del doesn't cascade to dependencies stream.progress('Removing xovi via Vellum...') const removeResult = await exec(client, `${DEVICE_PATHS.vellumBin} del qt-resource-rebuilder xovi-extensions xovi 2>&1`) if (removeResult.code !== 0) { + const output = removeResult.stdout + removeResult.stderr stream.error( 'Failed to remove xovi via Vellum', - undefined, - removeResult.stdout + removeResult.stderr, + parseVellumErrorHint(output), + output, ) return } @@ -360,7 +468,9 @@ export default function deviceXoviRoutes(app: FastifyInstance, _config: ServerCo await exec(client, DEVICE_PATHS.restartCmd) steps.push('Restarted xochitl') - stream.done({ steps, message: 'xovi removed successfully.' }) + const doneData: Record = { steps, message: 'xovi removed successfully.' } + if (warnings.length > 0) doneData.warnings = warnings + stream.done(doneData) } catch (err) { const friendly = formatSshError(err as Error) stream.error(friendly.message, friendly.hint, friendly.rawError) diff --git a/src/components/device/DeviceXoviCard.tsx b/src/components/device/DeviceXoviCard.tsx index 516d391..51d23e6 100644 --- a/src/components/device/DeviceXoviCard.tsx +++ b/src/components/device/DeviceXoviCard.tsx @@ -52,6 +52,7 @@ interface XoviDeviceStatus { vellumInstalled: boolean vellumVersion: string | null vellumReenableNeeded: boolean + supportedVersionRange: { min: string; max: string } | null } // ── Hooks ──────────────────────────────────────────────────────────────────── @@ -374,6 +375,7 @@ export function DeviceXoviCard({ deviceId, deviceName, configured, deviceModel, error={xoviStatus.error.message} hint={xoviStatus.error.hint} rawError={xoviStatus.error.rawError} + operationName="xovi-status" deviceModel={deviceModel} firmwareVersion={firmwareVersion} /> @@ -404,10 +406,16 @@ export function DeviceXoviCard({ deviceId, deviceName, configured, deviceModel, {status.firmwareVersion && (
- • + + {status.qmdVersion ? '\u2713' : '\u2717'} + Firmware {status.firmwareVersion} - {status.qmdVersion ? ` \u2192 extensions v${status.qmdVersion}` : ' (unsupported)'} + {status.qmdVersion + ? ` \u2192 extensions v${status.qmdVersion}` + : status.supportedVersionRange + ? ` (unsupported \u2014 bundled extensions cover ${status.supportedVersionRange.min}\u2013${status.supportedVersionRange.max})` + : ' (unsupported)'}
)} @@ -450,6 +458,7 @@ export function DeviceXoviCard({ deviceId, deviceName, configured, deviceModel, error={vellumOp.result.error} hint={vellumOp.result.hint} rawError={vellumOp.result.rawError} + operationName="vellum-install-xovi" deviceModel={deviceModel} firmwareVersion={firmwareVersion} /> @@ -481,9 +490,23 @@ export function DeviceXoviCard({ deviceId, deviceName, configured, deviceModel, {/* Extensions list */} {xoviReady && !hasAvailable && status.qmdVersion === null && ( -

- No extensions available for firmware {status.firmwareVersion}. Extensions may not yet support this version. -

+
+

+ No extensions available for firmware {status.firmwareVersion}. +

+

+ This app bundles extensions for firmware{' '} + {status.supportedVersionRange + ? `${status.supportedVersionRange.min}–${status.supportedVersionRange.max}` + : 'versions not detected'}.{' '} + QMD extensions target firmware-specific internals and cannot be safely used across versions. + Check for an app update or visit{' '} + + xovi on GitHub + {' '} + for the latest extensions. +

+
)} {xoviReady && hasAvailable && ( @@ -618,6 +641,7 @@ export function DeviceXoviCard({ deviceId, deviceName, configured, deviceModel, error={vellumOp.result.error} hint={vellumOp.result.hint} rawError={vellumOp.result.rawError} + operationName="vellum-remove-xovi" deviceModel={deviceModel} firmwareVersion={firmwareVersion} /> @@ -639,6 +663,8 @@ export function DeviceXoviCard({ deviceId, deviceName, configured, deviceModel, error={xoviOp.result.error} hint={xoviOp.result.hint} rawError={xoviOp.result.rawError} + details={xoviOp.result.details} + operationName="xovi-deploy" deviceModel={deviceModel} firmwareVersion={firmwareVersion} /> diff --git a/src/components/device/ErrorDetails.tsx b/src/components/device/ErrorDetails.tsx index bab0bbb..485cd96 100644 --- a/src/components/device/ErrorDetails.tsx +++ b/src/components/device/ErrorDetails.tsx @@ -8,6 +8,8 @@ export function ErrorDetails({ error, hint, rawError, + details, + operationName, deviceModel, firmwareVersion, className = 'device-op-result error', @@ -16,6 +18,8 @@ export function ErrorDetails({ error: string hint?: string rawError?: string + details?: string[] + operationName?: string deviceModel?: string firmwareVersion?: string className?: string @@ -25,17 +29,23 @@ export function ErrorDetails({ const [copied, setCopied] = useState(false) function handleCopy() { - const text = [ + const lines = [ `**Error:** ${error}`, - `**Raw:** ${rawError ?? 'N/A'}`, + ] + if (operationName) lines.push(`**Operation:** ${operationName}`) + lines.push( `**Hint:** ${hint ?? 'N/A'}`, + `**Raw:** ${rawError ?? 'N/A'}`, + ) + if (details?.length) lines.push(`**Details:**\n${details.map(d => ` - ${d}`).join('\n')}`) + lines.push( `**Device:** ${deviceModel ?? 'N/A'}`, `**Firmware:** ${firmwareVersion ?? 'N/A'}`, `**URL:** ${window.location.href}`, `**Time:** ${new Date().toISOString()}`, `**UA:** ${navigator.userAgent}`, - ].join('\n') - navigator.clipboard.writeText(text).then(() => { + ) + navigator.clipboard.writeText(lines.join('\n')).then(() => { setCopied(true) setTimeout(() => setCopied(false), 2000) }) @@ -45,6 +55,11 @@ export function ErrorDetails({

{error}

{hint &&

{hint}

} + {details && details.length > 0 && ( +
    + {details.map((d, i) =>
  • {d}
    • )} +
+ )} {rawError && rawError !== error && (