diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9914124 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# QMD extension files are text-like but checksummed — force LF to prevent +# CRLF conversion on Windows from breaking SHA-512 validation. +*.qmd text eol=lf + +# Keep shell scripts consistent +*.sh text eol=lf diff --git a/README.md b/README.md index 54f5b63..66410bd 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ A browser-based template editor for native reMarkable tablets. Browse, preview, ```bash git clone https://github.com/cuttlefisch/RemarkableCustomTemplates -cd remarkable_templates +cd RemarkableCustomTemplates docker compose up --build -d ``` @@ -23,6 +23,8 @@ Open **http://localhost:3000** in your browser. That's it. > **Port conflict?** `PORT=3001 docker compose up --build -d` > **Stop:** `docker compose down` · **Reset all data:** `docker compose down -v` +**Updating:** `git pull origin main && docker compose up --build -d` — your data is preserved across upgrades. See the [quickstart](docs/quickstart.md#updating-to-a-new-version) for details. + ## What You Can Do - **Browse & preview** all templates across reMarkable 1/2, Paper Pro, and Paper Pro Move resolutions diff --git a/docs/quickstart.md b/docs/quickstart.md index 0c204cc..bcf397c 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -11,7 +11,7 @@ Get remarkable-templates running and deploy custom templates to your reMarkable ```bash git clone https://github.com/cuttlefisch/RemarkableCustomTemplates -cd remarkable_templates +cd RemarkableCustomTemplates docker compose up --build -d ``` @@ -114,10 +114,24 @@ If something goes wrong, use the **Device & Sync** page to roll back: **Stop:** `docker compose down` -**Data persistence:** Templates, device config, and SSH keys are stored in a Docker volume and persist across restarts. +**Data persistence:** Templates, device config, and SSH keys are stored in a Docker volume and persist across restarts and upgrades. **Start fresh:** `docker compose down -v` removes the volume and all data. +## Updating to a new version + +Pull the latest code and rebuild. Your data (templates, device config, SSH keys, backups) is stored in a Docker volume and is preserved automatically. + +```bash +cd RemarkableCustomTemplates +git pull origin main +docker compose up --build -d +``` + +The `--build` flag rebuilds the Docker image with the latest code. The container restarts with the new version while keeping all your data intact. You can verify the build succeeded by checking for any errors in the build output — for example, xovi checksum validation runs during the build and will report any issues. + +> **Troubleshooting an upgrade:** If you see unexpected behavior after updating, check the [release notes](https://github.com/cuttlefisch/RemarkableCustomTemplates/releases) for breaking changes. If your data appears corrupted, you can start fresh with `docker compose down -v && docker compose up --build -d` — but this removes all local data, so back up first (see [Back up your templates](#8-back-up-your-templates)). + --- ## For developers diff --git a/docs/xovi-extensions.md b/docs/xovi-extensions.md index fee0e06..c4ddd00 100644 --- a/docs/xovi-extensions.md +++ b/docs/xovi-extensions.md @@ -99,6 +99,17 @@ When your reMarkable receives a firmware update: ## Troubleshooting +### Checksum validation errors during deploy + +If you see checksum errors when deploying extensions, update to the latest version: + +```bash +git pull origin main +docker compose up --build -d +``` + +This was fixed in [PR #8](https://github.com/cuttlefisch/RemarkableCustomTemplates/pull/8) — the root cause was line-ending conversion on Windows breaking file checksums. The fix normalizes line endings at all layers so checksums are consistent regardless of platform. + ### "xovi not installed" xovi and qt-resource-rebuilder must be installed on your device before this app can deploy extensions. If Vellum is on your device, click **Install xovi** in the app. Otherwise, install via SSH: diff --git a/scripts/generate-xovi-manifest.ts b/scripts/generate-xovi-manifest.ts index 439ae4a..49fbfe7 100644 --- a/scripts/generate-xovi-manifest.ts +++ b/scripts/generate-xovi-manifest.ts @@ -54,8 +54,19 @@ const EXTENSION_DEFS = { }, } +/** Normalize CRLF → LF so checksums are stable across platforms. */ +function normalizeLF(buf: Buffer): Buffer { + if (!buf.includes(0x0d)) return buf + const out: number[] = [] + for (let i = 0; i < buf.length; i++) { + if (buf[i] === 0x0d && i + 1 < buf.length && buf[i + 1] === 0x0a) continue + out.push(buf[i]!) + } + return Buffer.from(out) +} + function sha512(filePath: string): string { - const content = readFileSync(filePath) + const content = normalizeLF(readFileSync(filePath)) return createHash('sha512').update(content).digest('hex') } diff --git a/scripts/validate-xovi-checksums.ts b/scripts/validate-xovi-checksums.ts index 86158a4..f197499 100644 --- a/scripts/validate-xovi-checksums.ts +++ b/scripts/validate-xovi-checksums.ts @@ -2,6 +2,9 @@ * Validate SHA-512 checksums for bundled xovi extension QMD files. * Exits with code 1 if any checksum mismatches. * + * Normalizes line endings (CRLF → LF) before hashing so that checksums + * are stable across platforms even without .gitattributes protection. + * * Usage: npx tsx scripts/validate-xovi-checksums.ts */ @@ -14,6 +17,24 @@ const __dirname = dirname(fileURLToPath(import.meta.url)) const XOVI_DATA_DIR = resolve(__dirname, '../server/data/xovi-extensions') const manifestPath = resolve(XOVI_DATA_DIR, 'manifest.json') +/** Normalize CRLF → LF so checksums are stable across platforms. */ +function normalizeLF(buf: Buffer): Buffer { + // Fast path: no CR bytes at all + if (!buf.includes(0x0d)) return buf + // Strip \r from \r\n sequences + const out: number[] = [] + for (let i = 0; i < buf.length; i++) { + if (buf[i] === 0x0d && i + 1 < buf.length && buf[i + 1] === 0x0a) continue + out.push(buf[i]!) + } + return Buffer.from(out) +} + +/** Compute SHA-512 of a buffer after LF normalization. */ +export function sha512Normalized(buf: Buffer): string { + return createHash('sha512').update(normalizeLF(buf)).digest('hex') +} + if (!existsSync(manifestPath)) { console.error('manifest.json not found — run generate-xovi-manifest.ts first') process.exit(1) @@ -25,6 +46,7 @@ const manifest = JSON.parse(readFileSync(manifestPath, 'utf-8')) as { let ok = true let checked = 0 +let crlfDetected = false for (const [relPath, expectedHash] of Object.entries(manifest.checksums)) { const filePath = resolve(XOVI_DATA_DIR, relPath) @@ -33,8 +55,9 @@ for (const [relPath, expectedHash] of Object.entries(manifest.checksums)) { ok = false continue } - const content = readFileSync(filePath) - const actual = createHash('sha512').update(content).digest('hex') + const raw = readFileSync(filePath) + if (raw.includes(0x0d)) crlfDetected = true + const actual = sha512Normalized(raw) if (actual !== expectedHash) { console.error(`MISMATCH ${relPath}`) console.error(` expected: ${expectedHash}`) @@ -47,7 +70,21 @@ for (const [relPath, expectedHash] of Object.entries(manifest.checksums)) { if (ok) { console.log(`All ${checked} QMD file checksums verified.`) + if (crlfDetected) { + console.warn( + 'WARNING: Some QMD files contain CRLF line endings (normalized before hashing).\n' + + 'Run "git add --renormalize ." to fix, or ensure .gitattributes is present.', + ) + } } else { console.error('\nChecksum validation failed.') + if (crlfDetected) { + console.error( + 'HINT: CRLF line endings detected — this is a common cause of checksum mismatches.\n' + + 'Ensure .gitattributes marks *.qmd with eol=lf, then re-checkout:\n' + + ' git rm --cached server/data/xovi-extensions/**/*.qmd\n' + + ' git checkout -- server/data/xovi-extensions/', + ) + } process.exit(1) } diff --git a/server/__tests__/helpers/mockSshServer.ts b/server/__tests__/helpers/mockSshServer.ts index f0f96d4..fa7bdd0 100644 --- a/server/__tests__/helpers/mockSshServer.ts +++ b/server/__tests__/helpers/mockSshServer.ts @@ -176,9 +176,13 @@ function handleExec( return } - if (command.includes('mkdir -p') && command.includes('.ssh')) { - const sshDir = mapPath(fsRoot, '/home/root/.ssh') - mkdirSync(sshDir, { recursive: true }) + if (command.includes('mkdir -p')) { + // Extract the path argument + const mkdirMatch = command.match(/mkdir -p ([^\s;]+)/) + if (mkdirMatch) { + const dir = mapPath(fsRoot, mkdirMatch[1]) + mkdirSync(dir, { recursive: true }) + } channel.exit(0) channel.end() return @@ -227,6 +231,108 @@ function handleExec( return } + // Generic `test -f && echo ok || echo missing` pattern + const testFileMatch = command.match(/test -f ([^\s&|]+)\s*&&\s*echo ok\s*\|\|\s*echo missing/) + if (testFileMatch) { + const localPath = mapPath(fsRoot, testFileMatch[1]) + channel.write(existsSync(localPath) ? 'ok' : 'missing') + channel.exit(0) + channel.end() + return + } + + // Generic `test -x && echo ok || echo missing` pattern + const testExecMatch = command.match(/test -x ([^\s&|]+)\s*&&\s*echo ok\s*\|\|\s*echo missing/) + if (testExecMatch) { + const localPath = mapPath(fsRoot, testExecMatch[1]) + channel.write(existsSync(localPath) ? 'ok' : 'missing') + channel.exit(0) + channel.end() + return + } + + // Generic `test -f && || echo missing` — for vellum version check etc. + const testAndExecMatch = command.match(/test -f ([^\s&|]+)\s*&&/) + if (testAndExecMatch && !command.includes('echo ok')) { + const localPath = mapPath(fsRoot, testAndExecMatch[1]) + if (!existsSync(localPath)) { + channel.write('missing') + channel.exit(1) + channel.end() + return + } + // File exists — fall through to more specific handlers below + } + + // Vellum reenable status + if (command.includes('vellum') && command.includes('reenable status')) { + const marker = mapPath(fsRoot, '/home/root/.vellum/.reenable-needed') + channel.write(existsSync(marker) ? 'needed' : 'ok') + channel.exit(0) + channel.end() + return + } + + // Vellum --version + if (command.includes('vellum') && command.includes('--version')) { + const vellumBin = mapPath(fsRoot, '/home/root/.vellum/bin/vellum') + channel.write(existsSync(vellumBin) ? '1.0.0' : 'missing') + channel.exit(existsSync(vellumBin) ? 0 : 1) + channel.end() + return + } + + // Vellum add/del — always succeed (mock) + if (command.includes('vellum') && (command.includes(' add ') || command.includes(' del '))) { + channel.write('ok\n') + channel.exit(0) + channel.end() + return + } + + // rebuild_hashtable + if (command.includes('rebuild_hashtable')) { + const bin = mapPath(fsRoot, '/home/root/xovi/rebuild_hashtable') + if (existsSync(bin)) { + channel.write('Hashtable rebuilt successfully\n') + channel.exit(0) + } else { + channel.stderr.write('rebuild_hashtable: not found\n') + channel.exit(1) + } + channel.end() + return + } + + // rm -f — delete a file (best effort) + const rmMatch = command.match(/rm -f ([^\s;]+)/) + if (rmMatch) { + const localPath = mapPath(fsRoot, rmMatch[1]) + if (existsSync(localPath)) { + try { unlinkSync(localPath) } catch { /* best effort */ } + } + channel.exit(0) + channel.end() + return + } + + // rm -f pattern via shell (e.g. rm -f /path/*.qmd) + if (command.includes('rm -f') && command.includes('*.qmd')) { + // Extract dir from the glob pattern + const globMatch = command.match(/rm -f ([^\s*]+)\*\.qmd/) + if (globMatch) { + const dir = mapPath(fsRoot, globMatch[1]) + if (existsSync(dir)) { + for (const f of readdirSync(dir)) { + if (f.endsWith('.qmd')) { + try { unlinkSync(resolve(dir, f)) } catch { /* best effort */ } + } + } + } + } + // Fall through to default (will echo ok if piped) + } + // Default: succeed silently channel.exit(0) channel.end() diff --git a/server/__tests__/helpers/seedDeviceFs.ts b/server/__tests__/helpers/seedDeviceFs.ts index 600035c..65488f1 100644 --- a/server/__tests__/helpers/seedDeviceFs.ts +++ b/server/__tests__/helpers/seedDeviceFs.ts @@ -47,6 +47,32 @@ export function seedMethodsTemplates( ) } +/** Seed xovi/vellum filesystem structure for xovi integration tests. */ +export function seedXoviFs( + fsRoot: string, + opts?: { xovi?: boolean; qtRebuilder?: boolean; vellum?: boolean; reenableNeeded?: boolean }, +) { + const { xovi = true, qtRebuilder = true, vellum = true, reenableNeeded = false } = opts ?? {} + if (xovi) { + mkdirSync(resolve(fsRoot, 'home/root/xovi'), { recursive: true }) + writeFileSync(resolve(fsRoot, 'home/root/xovi/xovi.so'), 'fake-xovi') + writeFileSync(resolve(fsRoot, 'home/root/xovi/rebuild_hashtable'), '#!/bin/sh\necho ok') + } + if (qtRebuilder) { + mkdirSync(resolve(fsRoot, 'home/root/xovi/extensions.d'), { recursive: true }) + writeFileSync(resolve(fsRoot, 'home/root/xovi/extensions.d/qt-resource-rebuilder.so'), 'fake-qt') + mkdirSync(resolve(fsRoot, 'home/root/xovi/exthome/qt-resource-rebuilder'), { recursive: true }) + } + if (vellum) { + mkdirSync(resolve(fsRoot, 'home/root/.vellum/bin'), { recursive: true }) + writeFileSync(resolve(fsRoot, 'home/root/.vellum/bin/vellum'), '#!/bin/sh\necho 1.0.0') + } + // Marker file read by mock exec handler + if (reenableNeeded) { + writeFileSync(resolve(fsRoot, 'home/root/.vellum/.reenable-needed'), '') + } +} + /** Write classic templates (templates.json + .template files) to /usr/share/remarkable/templates/. */ export function seedClassicTemplates( fsRoot: string, diff --git a/server/__tests__/xoviExtensions.test.ts b/server/__tests__/xoviExtensions.test.ts index 8efdc4b..0613da6 100644 --- a/server/__tests__/xoviExtensions.test.ts +++ b/server/__tests__/xoviExtensions.test.ts @@ -1,11 +1,17 @@ // @vitest-environment node import { describe, it, expect, beforeEach } from 'vitest' +import { readFileSync, writeFileSync, unlinkSync } from 'node:fs' +import { resolve } from 'node:path' +import { tmpdir } from 'node:os' import { mapFirmwareToQmdVersion, getExtensionDefs, getQmdFilePath, validateExclusiveGroups, getSupportedVersions, + normalizeLF, + sha512Normalized, + verifyQmdChecksum, _resetManifestCache, } from '../lib/xoviExtensions.ts' @@ -143,3 +149,94 @@ describe('getSupportedVersions', () => { } }) }) + +// ── checksum utilities ────────────────────────────────────────────────────── + +describe('normalizeLF', () => { + it('returns the same buffer when no CR bytes present', () => { + const buf = Buffer.from('hello\nworld\n') + const result = normalizeLF(buf) + expect(result).toEqual(buf) + }) + + it('strips CR from CRLF sequences', () => { + const buf = Buffer.from('hello\r\nworld\r\n') + const result = normalizeLF(buf) + expect(result).toEqual(Buffer.from('hello\nworld\n')) + }) + + it('preserves bare CR that is not followed by LF', () => { + const buf = Buffer.from('hello\rworld\n') + const result = normalizeLF(buf) + expect(result).toEqual(Buffer.from('hello\rworld\n')) + }) + + it('handles mixed line endings', () => { + const buf = Buffer.from('a\r\nb\nc\r\nd\r') + const result = normalizeLF(buf) + expect(result).toEqual(Buffer.from('a\nb\nc\nd\r')) + }) + + it('handles empty buffer', () => { + const result = normalizeLF(Buffer.alloc(0)) + expect(result.length).toBe(0) + }) +}) + +describe('sha512Normalized', () => { + it('produces same hash for LF and CRLF versions of the same content', () => { + const lf = Buffer.from('line1\nline2\nline3\n') + const crlf = Buffer.from('line1\r\nline2\r\nline3\r\n') + expect(sha512Normalized(lf)).toBe(sha512Normalized(crlf)) + }) + + it('produces a 128-character hex string', () => { + const hash = sha512Normalized(Buffer.from('test')) + expect(hash).toMatch(/^[0-9a-f]{128}$/) + }) +}) + +describe('verifyQmdChecksum', () => { + it('verifies a known good QMD file', () => { + const result = verifyQmdChecksum('unlockMethodsContent', '3.26', + getQmdFilePath('unlockMethodsContent', '3.26')) + expect(result.ok).toBe(true) + expect(result.expected).toBeTruthy() + expect(result.actual).toBe(result.expected) + }) + + it('verifies all bundled QMD files pass checksum', () => { + for (const version of getSupportedVersions()) { + for (const def of getExtensionDefs()) { + let filePath: string + try { + filePath = getQmdFilePath(def.id, version) + } catch { + continue // extension not available in this version + } + const result = verifyQmdChecksum(def.id, version, filePath) + expect(result.ok, `${def.id}@${version}`).toBe(true) + } + } + }) + + it('detects a tampered file', () => { + const filePath = getQmdFilePath('unlockMethodsContent', '3.26') + const content = readFileSync(filePath) + const tampered = Buffer.concat([content, Buffer.from('X')]) + const tmpPath = resolve(tmpdir(), `tampered-${Date.now()}.qmd`) + writeFileSync(tmpPath, tampered) + try { + const result = verifyQmdChecksum('unlockMethodsContent', '3.26', tmpPath) + expect(result.ok).toBe(false) + expect(result.actual).not.toBe(result.expected) + } finally { + unlinkSync(tmpPath) + } + }) + + it('throws for unknown extension', () => { + expect(() => verifyQmdChecksum('nonexistent', '3.26', '/tmp/fake.qmd')) + .toThrow('Unknown extension') + }) +}) diff --git a/server/__tests__/xoviIntegration.test.ts b/server/__tests__/xoviIntegration.test.ts new file mode 100644 index 0000000..78943fc --- /dev/null +++ b/server/__tests__/xoviIntegration.test.ts @@ -0,0 +1,314 @@ +// @vitest-environment node +/** + * SSH integration tests for xovi extension routes. + * + * Uses an in-process ssh2 mock server backed by a real temp directory. + * Tests the full xovi lifecycle: status, deploy, remove, vellum install/remove. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from 'vitest' +import { mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { tmpdir } from 'node:os' +import { createApp } from '../app.ts' +import { resolveConfig, type ServerConfig } from '../config.ts' +import { writeDeviceStore } from '../lib/deviceStore.ts' +import type { DeviceConfig } from '../lib/ssh.ts' +import { startMockSshServer, type MockSshServer } from './helpers/mockSshServer.ts' +import { seedXoviFs } from './helpers/seedDeviceFs.ts' +import { parseNdjson } from './helpers/ndjsonHelper.ts' + +const TEST_TIMEOUT = 15_000 + +function makeConfig(): ServerConfig { + const base = resolve(tmpdir(), `xovi-integ-${Date.now()}-${Math.random().toString(36).slice(2)}`) + mkdirSync(resolve(base, 'public/templates/custom'), { recursive: true }) + mkdirSync(resolve(base, 'public/templates/debug'), { recursive: true }) + mkdirSync(resolve(base, 'public/templates/methods'), { recursive: true }) + mkdirSync(resolve(base, 'public/templates/samples'), { recursive: true }) + mkdirSync(resolve(base, 'remarkable_official_templates'), { recursive: true }) + mkdirSync(resolve(base, 'rm-methods-dist'), { recursive: true }) + mkdirSync(resolve(base, 'rm-methods-backups'), { recursive: true }) + mkdirSync(resolve(base, 'data/ssh'), { recursive: true }) + return resolveConfig({ dataDir: base, port: 0, production: false }) +} + +function seedDevice( + config: ServerConfig, + mockServer: MockSshServer, + opts?: { firmwareVersion?: string }, +): DeviceConfig { + const device: DeviceConfig = { + id: 'test-dev-1', + nickname: 'Test RM', + deviceIp: '127.0.0.1', + sshPort: mockServer.port, + authMethod: 'password', + sshPassword: 'test', + firmwareVersion: opts?.firmwareVersion ?? '3.26.1.2', + deviceModel: 'rmPP', + } + writeDeviceStore(config.deviceConfigPath, { + version: 2, + devices: [device], + activeDeviceId: device.id, + }) + return device +} + +describe('xovi SSH integration', () => { + let mockServer: MockSshServer + let config: ServerConfig + + beforeAll(async () => { + mockServer = await startMockSshServer() + }) + + afterAll(async () => { + await mockServer.close() + }) + + beforeEach(() => { + config = makeConfig() + mockServer.resetFs() + }) + + afterEach(() => { + rmSync(config.dataDir, { recursive: true, force: true }) + }) + + // ── xovi-status ────────────────────────────────────────────────────────── + + describe('POST /api/devices/:id/xovi-status', () => { + it('returns full status with xovi installed', async () => { + seedDevice(config, mockServer) + seedXoviFs(mockServer.fsRoot) + const app = await createApp(config) + const res = await app.inject({ method: 'POST', url: '/api/devices/test-dev-1/xovi-status' }) + expect(res.statusCode).toBe(200) + const body = JSON.parse(res.body) + expect(body.ok).toBe(true) + expect(body.xoviInstalled).toBe(true) + expect(body.qtRebuilderInstalled).toBe(true) + expect(body.vellumInstalled).toBe(true) + expect(body.qmdVersion).toBe('3.26') + expect(body.supportedVersionRange).toEqual({ min: '3.22', max: '3.26' }) + await app.close() + }, TEST_TIMEOUT) + + it('reports unsupported firmware when version is too new', async () => { + seedDevice(config, mockServer, { firmwareVersion: '3.99.0.0' }) + seedXoviFs(mockServer.fsRoot) + const app = await createApp(config) + const res = await app.inject({ method: 'POST', url: '/api/devices/test-dev-1/xovi-status' }) + expect(res.statusCode).toBe(200) + const body = JSON.parse(res.body) + expect(body.qmdVersion).toBeNull() + expect(body.supportedVersionRange).toEqual({ min: '3.22', max: '3.26' }) + await app.close() + }, TEST_TIMEOUT) + + it('detects vellum reenable needed', async () => { + seedDevice(config, mockServer) + seedXoviFs(mockServer.fsRoot, { reenableNeeded: true }) + const app = await createApp(config) + const res = await app.inject({ method: 'POST', url: '/api/devices/test-dev-1/xovi-status' }) + expect(res.statusCode).toBe(200) + const body = JSON.parse(res.body) + expect(body.vellumReenableNeeded).toBe(true) + await app.close() + }, TEST_TIMEOUT) + }) + + // ── xovi-deploy ────────────────────────────────────────────────────────── + + describe('POST /api/devices/:id/xovi-deploy', () => { + it('deploys extensions successfully', async () => { + seedDevice(config, mockServer) + seedXoviFs(mockServer.fsRoot) + const app = await createApp(config) + const res = await app.inject({ + method: 'POST', + url: '/api/devices/test-dev-1/xovi-deploy', + payload: { extensionIds: ['unlockMethodsContent'] }, + }) + expect(res.statusCode).toBe(200) + const events = parseNdjson(res.body) + const done = events.find(e => e.type === 'done') + expect(done).toBeTruthy() + expect((done as Record).extensions).toEqual(['unlockMethodsContent']) + await app.close() + }, TEST_TIMEOUT) + + it('blocks deploy when vellum reenable is needed', async () => { + seedDevice(config, mockServer) + seedXoviFs(mockServer.fsRoot, { reenableNeeded: true }) + const app = await createApp(config) + const res = await app.inject({ + method: 'POST', + url: '/api/devices/test-dev-1/xovi-deploy', + payload: { extensionIds: ['unlockMethodsContent'] }, + }) + expect(res.statusCode).toBe(200) + const events = parseNdjson(res.body) + const error = events.find(e => e.type === 'error') + expect(error).toBeTruthy() + expect((error as Record).error).toMatch(/firmware was updated/) + expect((error as Record).hint).toMatch(/vellum reenable/) + await app.close() + }, TEST_TIMEOUT) + + it('blocks deploy when xovi is not installed', async () => { + seedDevice(config, mockServer) + // No seedXoviFs — xovi not installed + const app = await createApp(config) + const res = await app.inject({ + method: 'POST', + url: '/api/devices/test-dev-1/xovi-deploy', + payload: { extensionIds: ['unlockMethodsContent'] }, + }) + expect(res.statusCode).toBe(200) + const events = parseNdjson(res.body) + const error = events.find(e => e.type === 'error') + expect(error).toBeTruthy() + expect((error as Record).error).toMatch(/not installed/) + await app.close() + }, TEST_TIMEOUT) + + it('returns 400 for unsupported firmware version with version range in hint', async () => { + seedDevice(config, mockServer, { firmwareVersion: '3.99.0.0' }) + seedXoviFs(mockServer.fsRoot) + const app = await createApp(config) + const res = await app.inject({ + method: 'POST', + url: '/api/devices/test-dev-1/xovi-deploy', + payload: { extensionIds: ['unlockMethodsContent'] }, + }) + expect(res.statusCode).toBe(400) + const body = JSON.parse(res.body) + expect(body.error).toMatch(/No extensions available/) + expect(body.hint).toMatch(/3\.22/) + expect(body.hint).toMatch(/3\.26/) + expect(body.hint).toMatch(/firmware-specific/) + await app.close() + }, TEST_TIMEOUT) + }) + + // ── xovi-remove ────────────────────────────────────────────────────────── + + describe('POST /api/devices/:id/xovi-remove', () => { + it('removes extensions successfully', async () => { + seedDevice(config, mockServer) + seedXoviFs(mockServer.fsRoot) + // Put a fake QMD file on "device" + const qmdDir = resolve(mockServer.fsRoot, 'home/root/xovi/exthome/qt-resource-rebuilder') + writeFileSync(resolve(qmdDir, 'unlockMethodsContent.qmd'), 'fake-qmd') + const app = await createApp(config) + const res = await app.inject({ + method: 'POST', + url: '/api/devices/test-dev-1/xovi-remove', + payload: { extensionIds: ['unlockMethodsContent'] }, + }) + expect(res.statusCode).toBe(200) + const events = parseNdjson(res.body) + const done = events.find(e => e.type === 'done') + expect(done).toBeTruthy() + expect((done as Record).removed).toEqual(['unlockMethodsContent']) + await app.close() + }, TEST_TIMEOUT) + }) + + // ── vellum-install-xovi ────────────────────────────────────────────────── + + describe('POST /api/devices/:id/vellum-install-xovi', () => { + it('installs xovi when vellum is ready', async () => { + seedDevice(config, mockServer) + seedXoviFs(mockServer.fsRoot, { xovi: false, qtRebuilder: false, vellum: true }) + const app = await createApp(config) + const res = await app.inject({ + method: 'POST', + url: '/api/devices/test-dev-1/vellum-install-xovi', + }) + expect(res.statusCode).toBe(200) + const events = parseNdjson(res.body) + const done = events.find(e => e.type === 'done') + expect(done).toBeTruthy() + expect((done as Record).message).toMatch(/installed successfully/) + await app.close() + }, TEST_TIMEOUT) + + it('blocks install when vellum reenable is needed', async () => { + seedDevice(config, mockServer) + seedXoviFs(mockServer.fsRoot, { vellum: true, reenableNeeded: true }) + const app = await createApp(config) + const res = await app.inject({ + method: 'POST', + url: '/api/devices/test-dev-1/vellum-install-xovi', + }) + expect(res.statusCode).toBe(200) + const events = parseNdjson(res.body) + const error = events.find(e => e.type === 'error') + expect(error).toBeTruthy() + expect((error as Record).error).toMatch(/re-enabled/) + await app.close() + }, TEST_TIMEOUT) + + it('errors when vellum is not installed', async () => { + seedDevice(config, mockServer) + // No vellum + const app = await createApp(config) + const res = await app.inject({ + method: 'POST', + url: '/api/devices/test-dev-1/vellum-install-xovi', + }) + expect(res.statusCode).toBe(200) + const events = parseNdjson(res.body) + const error = events.find(e => e.type === 'error') + expect(error).toBeTruthy() + expect((error as Record).error).toMatch(/not installed/) + await app.close() + }, TEST_TIMEOUT) + }) + + // ── vellum-remove-xovi ─────────────────────────────────────────────────── + + describe('POST /api/devices/:id/vellum-remove-xovi', () => { + it('cleans QMD files before uninstalling xovi', async () => { + seedDevice(config, mockServer) + seedXoviFs(mockServer.fsRoot) + // Put QMD files on "device" + const qmdDir = resolve(mockServer.fsRoot, 'home/root/xovi/exthome/qt-resource-rebuilder') + writeFileSync(resolve(qmdDir, 'unlockMethodsContent.qmd'), 'fake-qmd') + const app = await createApp(config) + const res = await app.inject({ + method: 'POST', + url: '/api/devices/test-dev-1/vellum-remove-xovi', + }) + expect(res.statusCode).toBe(200) + const events = parseNdjson(res.body) + const done = events.find(e => e.type === 'done') + expect(done).toBeTruthy() + expect((done as Record).message).toMatch(/removed successfully/) + // Verify steps include expected operations + const steps = (done as Record).steps as string[] + expect(steps).toBeDefined() + expect(steps.some(s => s.includes('Removed xovi') || s.includes('Cleaned QMD'))).toBe(true) + await app.close() + }, TEST_TIMEOUT) + + it('errors when vellum is not installed', async () => { + seedDevice(config, mockServer) + // No vellum + const app = await createApp(config) + const res = await app.inject({ + method: 'POST', + url: '/api/devices/test-dev-1/vellum-remove-xovi', + }) + expect(res.statusCode).toBe(200) + const events = parseNdjson(res.body) + const error = events.find(e => e.type === 'error') + expect(error).toBeTruthy() + expect((error as Record).error).toMatch(/not installed/) + await app.close() + }, TEST_TIMEOUT) + }) +}) diff --git a/server/__tests__/xoviRoutes.test.ts b/server/__tests__/xoviRoutes.test.ts index 6fdfcd0..8183bc6 100644 --- a/server/__tests__/xoviRoutes.test.ts +++ b/server/__tests__/xoviRoutes.test.ts @@ -6,6 +6,7 @@ import { tmpdir } from 'node:os' import { createApp } from '../app.ts' import { resolveConfig, type ServerConfig } from '../config.ts' import { writeDeviceStore } from '../lib/deviceStore.ts' +import { parseVellumErrorHint } from '../routes/device/xovi.ts' function makeConfig(): ServerConfig { const base = resolve(tmpdir(), `xoviroutes-test-${Date.now()}-${Math.random().toString(36).slice(2)}`) @@ -237,3 +238,52 @@ describe('xovi routes', () => { }) }) }) + +// ── parseVellumErrorHint ────────────────────────────────────────────────── + +describe('parseVellumErrorHint', () => { + it('detects checksum errors', () => { + const hint = parseVellumErrorHint('Error: checksum mismatch for package xovi') + expect(hint).toMatch(/checksum/) + expect(hint).toMatch(/vellum update/) + }) + + it('detects hash mismatch errors', () => { + const hint = parseVellumErrorHint('hash mismatch: expected abc123, got def456') + expect(hint).toMatch(/checksum/) + }) + + it('detects integrity errors', () => { + const hint = parseVellumErrorHint('integrity verification failed') + expect(hint).toMatch(/checksum/) + }) + + it('detects reenable-needed errors', () => { + const hint = parseVellumErrorHint('Please reenable vellum after firmware update') + expect(hint).toMatch(/re-enabled/) + expect(hint).toMatch(/vellum reenable/) + }) + + it('detects network errors', () => { + expect(parseVellumErrorHint('could not resolve host packages.vellum.dev')).toMatch(/internet/) + expect(parseVellumErrorHint('connection refused')).toMatch(/internet/) + expect(parseVellumErrorHint('network unreachable')).toMatch(/internet/) + expect(parseVellumErrorHint('request timeout')).toMatch(/internet/) + }) + + it('detects missing package errors', () => { + const hint = parseVellumErrorHint('Error: no such package xovi-extensions') + expect(hint).toMatch(/not found/) + }) + + it('detects disk space errors', () => { + expect(parseVellumErrorHint('ENOSPC: no space left on device')).toMatch(/disk space/) + expect(parseVellumErrorHint('No space left on device')).toMatch(/disk space/) + }) + + it('returns generic hint for unknown errors', () => { + const hint = parseVellumErrorHint('some unexpected error output') + expect(hint).toMatch(/internet access/) + expect(hint).toMatch(/manually/) + }) +}) diff --git a/server/lib/xoviExtensions.ts b/server/lib/xoviExtensions.ts index 1ec891f..297f839 100644 --- a/server/lib/xoviExtensions.ts +++ b/server/lib/xoviExtensions.ts @@ -5,6 +5,7 @@ * Pure logic (except checkXoviStatus which uses SSH/SFTP). */ +import { createHash } from 'node:crypto' import { readFileSync, existsSync } from 'node:fs' import { resolve, dirname } from 'node:path' import { fileURLToPath } from 'node:url' @@ -77,6 +78,44 @@ export function _resetManifestCache(): void { _manifest = null } +// ── checksum utilities ────────────────────────────────────────────────────── + +/** Normalize CRLF → LF so checksums are stable across platforms. */ +export function normalizeLF(buf: Buffer): Buffer { + if (!buf.includes(0x0d)) return buf + const out: number[] = [] + for (let i = 0; i < buf.length; i++) { + if (buf[i] === 0x0d && i + 1 < buf.length && buf[i + 1] === 0x0a) continue + out.push(buf[i]!) + } + return Buffer.from(out) +} + +/** Compute SHA-512 of a buffer after LF normalization. */ +export function sha512Normalized(buf: Buffer): string { + return createHash('sha512').update(normalizeLF(buf)).digest('hex') +} + +/** + * Verify a QMD file's integrity against the manifest checksum. + * Returns `{ ok, expected, actual }`. If the manifest has no entry + * for the given key, `expected` is null (no checksum to verify against). + */ +export function verifyQmdChecksum( + extensionId: string, + qmdVersion: string, + filePath: string, +): { ok: boolean; expected: string | null; actual: string } { + const manifest = loadManifest() + const def = manifest.extensions[extensionId] + if (!def) throw new Error(`Unknown extension: ${extensionId}`) + const key = `${qmdVersion}/${def.filename}` + const expected = manifest.checksums[key] ?? null + const content = readFileSync(filePath) + const actual = sha512Normalized(content) + return { ok: expected === null || actual === expected, expected, actual } +} + // ── public API ─────────────────────────────────────────────────────────────── /** diff --git a/server/routes/device/xovi.ts b/server/routes/device/xovi.ts index d4de4ff..0905e44 100644 --- a/server/routes/device/xovi.ts +++ b/server/routes/device/xovi.ts @@ -19,11 +19,50 @@ import { checkXoviStatus, getExtensionDefs, getQmdFilePath, + getSupportedVersions, mapFirmwareToQmdVersion, validateExclusiveGroups, + verifyQmdChecksum, DEVICE_PATHS, } from '../../lib/xoviExtensions.ts' +/** + * Parse Vellum command output for known error patterns and return a targeted hint. + */ +export function parseVellumErrorHint(output: string): string { + const lower = output.toLowerCase() + + if (lower.includes('checksum') || lower.includes('hash mismatch') || lower.includes('integrity')) { + return 'Package checksum verification failed. This usually means the package repository is out of sync. ' + + 'Try: SSH into the device and run "vellum update" to refresh package metadata, then retry.' + } + + if (lower.includes('reenable') || lower.includes('re-enable')) { + return 'Vellum needs to be re-enabled after a firmware update. ' + + 'SSH into your device and run: vellum reenable' + } + + if (lower.includes('no route') || lower.includes('could not resolve') || lower.includes('network') + || lower.includes('connection refused') || lower.includes('timeout')) { + return 'The device cannot reach the package server. ' + + 'Check that the device has a working internet connection (Settings → Wi-Fi), then retry.' + } + + if (lower.includes('not found') || lower.includes('no such package')) { + return 'One or more packages were not found in the Vellum repository. ' + + 'The package names may have changed. Check https://github.com/vellum-dev/vellum for updates.' + } + + if (lower.includes('disk') || lower.includes('no space') || lower.includes('enospc')) { + return 'The device is running low on disk space. ' + + 'Free up space by removing unused notebooks or files, then retry.' + } + + return 'Check that the device has internet access and try again. ' + + 'If the problem persists, SSH into the device and run the command manually: ' + + 'vellum add xovi xovi-extensions qt-resource-rebuilder' +} + export default function deviceXoviRoutes(app: FastifyInstance, _config: ServerConfig) { // ── POST /api/devices/:id/xovi-status ────────────────────────────────────── app.post<{ Params: { id: string } }>('/api/devices/:id/xovi-status', async (request, reply) => { @@ -38,7 +77,14 @@ export default function deviceXoviRoutes(app: FastifyInstance, _config: ServerCo client = await connect(deviceConfig) const sftp = await getSftp(client) const status = await checkXoviStatus(client, sftp, deviceConfig.firmwareVersion ?? null) - return reply.send({ ok: true, ...status }) + const supported = getSupportedVersions() + return reply.send({ + ok: true, + ...status, + supportedVersionRange: supported.length > 0 + ? { min: supported[0], max: supported[supported.length - 1] } + : null, + }) } catch (err) { const friendly = formatSshError(err as Error) return reply.status(500).send({ error: friendly.message, hint: friendly.hint, rawError: friendly.rawError }) @@ -84,9 +130,13 @@ export default function deviceXoviRoutes(app: FastifyInstance, _config: ServerCo } const qmdVersion = mapFirmwareToQmdVersion(fw) if (!qmdVersion) { + const supported = getSupportedVersions() + const range = supported.length > 0 ? `${supported[0]}–${supported[supported.length - 1]}` : 'none' return reply.status(400).send({ error: `No extensions available for firmware ${fw}`, - hint: 'Extensions may not yet support this firmware version.', + hint: `This app bundles extensions for firmware ${range}. ` + + 'QMD extensions are firmware-specific and cannot be safely used across versions. ' + + 'Check for an app update, or visit https://github.com/rmitchellscott/xovi for the latest extensions.', }) } @@ -105,6 +155,29 @@ export default function deviceXoviRoutes(app: FastifyInstance, _config: ServerCo } } + // Verify QMD file integrity before connecting to device + const checksumFailures: string[] = [] + for (const { extensionId, localPath } of filePaths) { + try { + const result = verifyQmdChecksum(extensionId, qmdVersion, localPath) + if (!result.ok) { + checksumFailures.push( + `${extensionId}: expected ${result.expected?.slice(0, 16)}..., got ${result.actual.slice(0, 16)}...`, + ) + } + } catch { + checksumFailures.push(`${extensionId}: unable to verify checksum`) + } + } + if (checksumFailures.length > 0) { + return reply.status(400).send({ + error: 'QMD file integrity check failed', + hint: 'Extension files may be corrupted (CRLF line ending conversion is a common cause). ' + + 'Re-clone the repository or run: git rm --cached server/data/xovi-extensions/**/*.qmd && git checkout -- server/data/xovi-extensions/', + details: checksumFailures, + }) + } + const stream = createNdjsonStream(reply) let client: Awaited> | null = null @@ -136,6 +209,21 @@ export default function deviceXoviRoutes(app: FastifyInstance, _config: ServerCo ) return } + // Check vellum reenable status — after a firmware update, QMD patches + // target stale symbol IDs and must not be deployed until the user re-enables + // vellum and installs QMDs matching the new firmware. + const vellumExists = await exec(client, `test -f ${DEVICE_PATHS.vellumBin} && echo ok || echo missing`) + if (vellumExists.stdout.trim() === 'ok') { + const reenableResult = await exec(client, `${DEVICE_PATHS.vellumBin} reenable status 2>/dev/null`) + if (reenableResult.stdout.trim() === 'needed') { + stream.error( + 'Deploy blocked: firmware was updated since xovi was installed', + 'QMD extensions target firmware-specific symbols and cannot be safely deployed until Vellum is re-enabled. ' + + 'SSH into your device and run: vellum reenable — then check xovi status again.', + ) + return + } + } steps.push('xovi prerequisites verified') // Ensure QMD directory exists @@ -229,7 +317,7 @@ export default function deviceXoviRoutes(app: FastifyInstance, _config: ServerCo if (rebuildResult.code !== 0) { stream.error( 'rebuild_hashtable failed after removal', - `Exit code ${rebuildResult.code}`, + `Exit code ${rebuildResult.code}. Try running it manually: ${DEVICE_PATHS.rebuildCmd}`, rebuildResult.stderr, ) return @@ -296,10 +384,11 @@ export default function deviceXoviRoutes(app: FastifyInstance, _config: ServerCo stream.progress('Installing xovi via Vellum (downloading packages)...') const installResult = await exec(client, `${DEVICE_PATHS.vellumBin} add xovi xovi-extensions qt-resource-rebuilder 2>&1`) if (installResult.code !== 0) { + const output = installResult.stdout + installResult.stderr stream.error( 'Failed to install xovi via Vellum', - 'Check that the device has internet access and try again.', - installResult.stdout + installResult.stderr, + parseVellumErrorHint(output), + output, ) return } @@ -342,14 +431,33 @@ export default function deviceXoviRoutes(app: FastifyInstance, _config: ServerCo return } + // Clean up deployed QMD files BEFORE vellum del — after removal, + // rebuild_hashtable won't be available so we must rebuild now. + const warnings: string[] = [] + stream.progress('Removing deployed QMD extensions...') + const qmdClean = await exec(client, `rm -f ${DEVICE_PATHS.qmdDir}/*.qmd 2>/dev/null; echo ok`) + if (qmdClean.stdout.trim() === 'ok') { + const hasRebuilder = await exec(client, `test -x /home/root/xovi/rebuild_hashtable && echo ok || echo missing`) + if (hasRebuilder.stdout.trim() === 'ok') { + stream.progress('Rebuilding hashtable before uninstall...') + const rebuildResult = await exec(client, DEVICE_PATHS.rebuildCmd) + if (rebuildResult.code !== 0) { + warnings.push(`rebuild_hashtable returned exit code ${rebuildResult.code} during cleanup (continuing with uninstall)`) + } else { + steps.push('Cleaned QMD extensions and rebuilt hashtable') + } + } + } + // Remove all three packages explicitly — vellum del doesn't cascade to dependencies stream.progress('Removing xovi via Vellum...') const removeResult = await exec(client, `${DEVICE_PATHS.vellumBin} del qt-resource-rebuilder xovi-extensions xovi 2>&1`) if (removeResult.code !== 0) { + const output = removeResult.stdout + removeResult.stderr stream.error( 'Failed to remove xovi via Vellum', - undefined, - removeResult.stdout + removeResult.stderr, + parseVellumErrorHint(output), + output, ) return } @@ -360,7 +468,9 @@ export default function deviceXoviRoutes(app: FastifyInstance, _config: ServerCo await exec(client, DEVICE_PATHS.restartCmd) steps.push('Restarted xochitl') - stream.done({ steps, message: 'xovi removed successfully.' }) + const doneData: Record = { steps, message: 'xovi removed successfully.' } + if (warnings.length > 0) doneData.warnings = warnings + stream.done(doneData) } catch (err) { const friendly = formatSshError(err as Error) stream.error(friendly.message, friendly.hint, friendly.rawError) diff --git a/src/components/device/DeviceXoviCard.tsx b/src/components/device/DeviceXoviCard.tsx index 516d391..51d23e6 100644 --- a/src/components/device/DeviceXoviCard.tsx +++ b/src/components/device/DeviceXoviCard.tsx @@ -52,6 +52,7 @@ interface XoviDeviceStatus { vellumInstalled: boolean vellumVersion: string | null vellumReenableNeeded: boolean + supportedVersionRange: { min: string; max: string } | null } // ── Hooks ──────────────────────────────────────────────────────────────────── @@ -374,6 +375,7 @@ export function DeviceXoviCard({ deviceId, deviceName, configured, deviceModel, error={xoviStatus.error.message} hint={xoviStatus.error.hint} rawError={xoviStatus.error.rawError} + operationName="xovi-status" deviceModel={deviceModel} firmwareVersion={firmwareVersion} /> @@ -404,10 +406,16 @@ export function DeviceXoviCard({ deviceId, deviceName, configured, deviceModel, {status.firmwareVersion && (
- • + + {status.qmdVersion ? '\u2713' : '\u2717'} + Firmware {status.firmwareVersion} - {status.qmdVersion ? ` \u2192 extensions v${status.qmdVersion}` : ' (unsupported)'} + {status.qmdVersion + ? ` \u2192 extensions v${status.qmdVersion}` + : status.supportedVersionRange + ? ` (unsupported \u2014 bundled extensions cover ${status.supportedVersionRange.min}\u2013${status.supportedVersionRange.max})` + : ' (unsupported)'}
)} @@ -450,6 +458,7 @@ export function DeviceXoviCard({ deviceId, deviceName, configured, deviceModel, error={vellumOp.result.error} hint={vellumOp.result.hint} rawError={vellumOp.result.rawError} + operationName="vellum-install-xovi" deviceModel={deviceModel} firmwareVersion={firmwareVersion} /> @@ -481,9 +490,23 @@ export function DeviceXoviCard({ deviceId, deviceName, configured, deviceModel, {/* Extensions list */} {xoviReady && !hasAvailable && status.qmdVersion === null && ( -

- No extensions available for firmware {status.firmwareVersion}. Extensions may not yet support this version. -

+
+

+ No extensions available for firmware {status.firmwareVersion}. +

+

+ This app bundles extensions for firmware{' '} + {status.supportedVersionRange + ? `${status.supportedVersionRange.min}–${status.supportedVersionRange.max}` + : 'versions not detected'}.{' '} + QMD extensions target firmware-specific internals and cannot be safely used across versions. + Check for an app update or visit{' '} + + xovi on GitHub + {' '} + for the latest extensions. +

+
)} {xoviReady && hasAvailable && ( @@ -618,6 +641,7 @@ export function DeviceXoviCard({ deviceId, deviceName, configured, deviceModel, error={vellumOp.result.error} hint={vellumOp.result.hint} rawError={vellumOp.result.rawError} + operationName="vellum-remove-xovi" deviceModel={deviceModel} firmwareVersion={firmwareVersion} /> @@ -639,6 +663,8 @@ export function DeviceXoviCard({ deviceId, deviceName, configured, deviceModel, error={xoviOp.result.error} hint={xoviOp.result.hint} rawError={xoviOp.result.rawError} + details={xoviOp.result.details} + operationName="xovi-deploy" deviceModel={deviceModel} firmwareVersion={firmwareVersion} /> diff --git a/src/components/device/ErrorDetails.tsx b/src/components/device/ErrorDetails.tsx index bab0bbb..485cd96 100644 --- a/src/components/device/ErrorDetails.tsx +++ b/src/components/device/ErrorDetails.tsx @@ -8,6 +8,8 @@ export function ErrorDetails({ error, hint, rawError, + details, + operationName, deviceModel, firmwareVersion, className = 'device-op-result error', @@ -16,6 +18,8 @@ export function ErrorDetails({ error: string hint?: string rawError?: string + details?: string[] + operationName?: string deviceModel?: string firmwareVersion?: string className?: string @@ -25,17 +29,23 @@ export function ErrorDetails({ const [copied, setCopied] = useState(false) function handleCopy() { - const text = [ + const lines = [ `**Error:** ${error}`, - `**Raw:** ${rawError ?? 'N/A'}`, + ] + if (operationName) lines.push(`**Operation:** ${operationName}`) + lines.push( `**Hint:** ${hint ?? 'N/A'}`, + `**Raw:** ${rawError ?? 'N/A'}`, + ) + if (details?.length) lines.push(`**Details:**\n${details.map(d => ` - ${d}`).join('\n')}`) + lines.push( `**Device:** ${deviceModel ?? 'N/A'}`, `**Firmware:** ${firmwareVersion ?? 'N/A'}`, `**URL:** ${window.location.href}`, `**Time:** ${new Date().toISOString()}`, `**UA:** ${navigator.userAgent}`, - ].join('\n') - navigator.clipboard.writeText(text).then(() => { + ) + navigator.clipboard.writeText(lines.join('\n')).then(() => { setCopied(true) setTimeout(() => setCopied(false), 2000) }) @@ -45,6 +55,11 @@ export function ErrorDetails({

{error}

{hint &&

{hint}

} + {details && details.length > 0 && ( +
    + {details.map((d, i) =>
  • {d}
    • )} +
+ )} {rawError && rawError !== error && (