From f19510430820941f242a88a3f6e79ee9d6bf638e Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Fri, 27 Mar 2026 23:45:38 -0700 Subject: [PATCH] feat: add musl libc-test conformance suite and rename posix-* to os-test-* MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename existing POSIX conformance infrastructure to os-test-* to reflect the actual test suite name (Sortix os-test). Add musl libc-test as a second conformance suite that tests kernel-level behavior (file locking, stat edge cases, socket operations) rather than just libc function correctness. - Rename posix-conformance → os-test-conformance across all files - Rename posix-exclusion-schema → conformance-exclusion-schema (shared) - Add import, build, test, validate, and report infrastructure for libc-test - 75 libc-test programs compile for WASM, 69 pass (92.0%) - 6 exclusions: 4 wasm-limitation (dlopen/TLS), 1 wasi-gap (statvfs), 1 implementation-gap (strptime) --- .github/workflows/libc-test-conformance.yml | 122 ++++ ...onformance.yml => os-test-conformance.yml} | 36 +- CLAUDE.md | 4 +- docs-internal/posix-gaps-audit.md | 85 +++ docs/docs.json | 3 +- docs/libc-test-conformance-report.mdx | 60 ++ ...ort.mdx => os-test-conformance-report.mdx} | 12 +- docs/posix-compatibility.md | 2 +- libc-test-conformance-report.json | 552 ++++++++++++++++++ native/wasmvm/c/.gitignore | 3 + native/wasmvm/c/Makefile | 94 +++ ...rt.json => os-test-conformance-report.json | 0 .../wasmvm/test/libc-test-conformance.test.ts | 383 ++++++++++++ .../wasmvm/test/libc-test-exclusions.json | 43 ++ ...ce.test.ts => os-test-conformance.test.ts} | 8 +- ...xclusions.json => os-test-exclusions.json} | 0 ...ema.ts => conformance-exclusion-schema.ts} | 9 +- scripts/generate-libc-test-report.ts | 188 ++++++ ...x-report.ts => generate-os-test-report.ts} | 26 +- scripts/import-libc-test.ts | 223 +++++++ scripts/import-os-test.ts | 12 +- scripts/validate-libc-test-exclusions.ts | 118 ++++ ...ions.ts => validate-os-test-exclusions.ts} | 12 +- 23 files changed, 1933 insertions(+), 62 deletions(-) create mode 100644 .github/workflows/libc-test-conformance.yml rename .github/workflows/{posix-conformance.yml => os-test-conformance.yml} (77%) create mode 100644 docs-internal/posix-gaps-audit.md create mode 100644 docs/libc-test-conformance-report.mdx rename docs/{posix-conformance-report.mdx => os-test-conformance-report.mdx} (75%) create mode 100644 libc-test-conformance-report.json rename posix-conformance-report.json => os-test-conformance-report.json (100%) create mode 100644 packages/wasmvm/test/libc-test-conformance.test.ts create mode 100644 packages/wasmvm/test/libc-test-exclusions.json rename packages/wasmvm/test/{posix-conformance.test.ts => os-test-conformance.test.ts} (98%) rename packages/wasmvm/test/{posix-exclusions.json => os-test-exclusions.json} (100%) rename scripts/{posix-exclusion-schema.ts => conformance-exclusion-schema.ts} (87%) create mode 100644 scripts/generate-libc-test-report.ts rename scripts/{generate-posix-report.ts => generate-os-test-report.ts} (88%) create mode 100644 scripts/import-libc-test.ts create mode 100644 scripts/validate-libc-test-exclusions.ts rename scripts/{validate-posix-exclusions.ts => validate-os-test-exclusions.ts} (92%) diff --git a/.github/workflows/libc-test-conformance.yml b/.github/workflows/libc-test-conformance.yml new file mode 100644 index 00000000..79515c2d --- /dev/null +++ b/.github/workflows/libc-test-conformance.yml @@ -0,0 +1,122 @@ +name: libc-test Conformance + +on: + push: + branches: + - main + paths: + - "native/wasmvm/**" + - "packages/wasmvm/**" + - "scripts/validate-libc-test-exclusions.ts" + - "scripts/generate-libc-test-report.ts" + - "scripts/import-libc-test.ts" + - "scripts/conformance-exclusion-schema.ts" + - ".github/workflows/libc-test-conformance.yml" + pull_request: + branches: + - main + paths: + - "native/wasmvm/**" + - "packages/wasmvm/**" + - "scripts/validate-libc-test-exclusions.ts" + - "scripts/generate-libc-test-report.ts" + - "scripts/import-libc-test.ts" + - "scripts/conformance-exclusion-schema.ts" + - ".github/workflows/libc-test-conformance.yml" + +jobs: + libc-test-conformance: + name: libc-test Conformance (musl kernel behavior) + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # --- Rust / WASM build --- + - name: Set up Rust toolchain + uses: dtolnay/rust-toolchain@nightly + with: + toolchain: nightly-2026-03-01 + targets: wasm32-wasip1 + components: rust-src + + - name: Install wasm-opt (binaryen) + run: sudo apt-get update && sudo apt-get install -y binaryen + + - name: Cache WASM build artifacts + uses: actions/cache@v4 + with: + path: | + native/wasmvm/target + native/wasmvm/vendor + key: wasm-${{ runner.os }}-${{ hashFiles('native/wasmvm/Cargo.lock', 'native/wasmvm/rust-toolchain.toml') }} + + - name: Build WASM binaries + run: cd native/wasmvm && make wasm + + # --- C toolchain (wasi-sdk + patched sysroot) --- + - name: Cache wasi-sdk + id: cache-wasi-sdk + uses: actions/cache@v4 + with: + path: native/wasmvm/c/vendor/wasi-sdk + key: wasi-sdk-25-${{ runner.os }}-${{ runner.arch }} + + - name: Download wasi-sdk + if: steps.cache-wasi-sdk.outputs.cache-hit != 'true' + run: make -C native/wasmvm/c wasi-sdk + + - name: Cache patched wasi-libc sysroot + id: cache-sysroot + uses: actions/cache@v4 + with: + path: | + native/wasmvm/c/sysroot + native/wasmvm/c/vendor/wasi-libc + key: wasi-libc-sysroot-${{ runner.os }}-${{ hashFiles('native/wasmvm/patches/wasi-libc/*.patch', 'native/wasmvm/scripts/patch-wasi-libc.sh') }} + + - name: Build patched wasi-libc sysroot + if: steps.cache-sysroot.outputs.cache-hit != 'true' + run: make -C native/wasmvm/c sysroot + + # --- Build libc-test (WASM + native) --- + - name: Build libc-test binaries (WASM + native) + run: make -C native/wasmvm/c libc-test libc-test-native + + # --- Node.js / TypeScript --- + - name: Set up pnpm + uses: pnpm/action-setup@v4 + with: + version: 8.15.6 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + # --- Run conformance tests --- + - name: Run libc-test conformance tests + run: pnpm vitest run packages/wasmvm/test/libc-test-conformance.test.ts + + - name: Validate exclusion list + run: pnpm tsx scripts/validate-libc-test-exclusions.ts + + # --- Generate report --- + - name: Generate conformance report MDX + if: always() + run: pnpm tsx scripts/generate-libc-test-report.ts + + # --- Upload artifacts --- + - name: Upload conformance report + if: always() + uses: actions/upload-artifact@v4 + with: + name: libc-test-conformance-report + path: | + libc-test-conformance-report.json + docs/libc-test-conformance-report.mdx diff --git a/.github/workflows/posix-conformance.yml b/.github/workflows/os-test-conformance.yml similarity index 77% rename from .github/workflows/posix-conformance.yml rename to .github/workflows/os-test-conformance.yml index cd92092e..af9aa02c 100644 --- a/.github/workflows/posix-conformance.yml +++ b/.github/workflows/os-test-conformance.yml @@ -1,4 +1,4 @@ -name: POSIX Conformance +name: os-test Conformance on: push: @@ -7,26 +7,26 @@ on: paths: - "native/wasmvm/**" - "packages/wasmvm/**" - - "scripts/validate-posix-exclusions.ts" - - "scripts/generate-posix-report.ts" + - "scripts/validate-os-test-exclusions.ts" + - "scripts/generate-os-test-report.ts" - "scripts/import-os-test.ts" - - "scripts/posix-exclusion-schema.ts" - - ".github/workflows/posix-conformance.yml" + - "scripts/conformance-exclusion-schema.ts" + - ".github/workflows/os-test-conformance.yml" pull_request: branches: - main paths: - "native/wasmvm/**" - "packages/wasmvm/**" - - "scripts/validate-posix-exclusions.ts" - - "scripts/generate-posix-report.ts" + - "scripts/validate-os-test-exclusions.ts" + - "scripts/generate-os-test-report.ts" - "scripts/import-os-test.ts" - - "scripts/posix-exclusion-schema.ts" - - ".github/workflows/posix-conformance.yml" + - "scripts/conformance-exclusion-schema.ts" + - ".github/workflows/os-test-conformance.yml" jobs: - posix-conformance: - name: POSIX Conformance (os-test) + os-test-conformance: + name: os-test Conformance (POSIX.1-2024) runs-on: ubuntu-latest steps: - name: Checkout repository @@ -100,23 +100,23 @@ jobs: run: pnpm install --frozen-lockfile # --- Run conformance tests --- - - name: Run POSIX conformance tests - run: pnpm vitest run packages/wasmvm/test/posix-conformance.test.ts + - name: Run os-test conformance tests + run: pnpm vitest run packages/wasmvm/test/os-test-conformance.test.ts - name: Validate exclusion list - run: pnpm tsx scripts/validate-posix-exclusions.ts + run: pnpm tsx scripts/validate-os-test-exclusions.ts # --- Generate report --- - name: Generate conformance report MDX if: always() - run: pnpm tsx scripts/generate-posix-report.ts + run: pnpm tsx scripts/generate-os-test-report.ts # --- Upload artifacts --- - name: Upload conformance report if: always() uses: actions/upload-artifact@v4 with: - name: posix-conformance-report + name: os-test-conformance-report path: | - posix-conformance-report.json - docs/posix-conformance-report.mdx + os-test-conformance-report.json + docs/os-test-conformance-report.mdx diff --git a/CLAUDE.md b/CLAUDE.md index bde44f99..23bc53e7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -35,7 +35,7 @@ ### POSIX Conformance Test Integrity - **no test-only workarounds** — if a C override fixes broken libc behavior (fcntl, realloc, strfmon, etc.), it MUST go in the patched sysroot (`native/wasmvm/patches/wasi-libc/`) so all WASM programs get the fix; never link overrides only into test binaries — that inflates conformance numbers while real users still hit the bug -- **never replace upstream test source files** — if an os-test `.c` file fails due to a platform difference (e.g. `sizeof(long)`), exclude it via `posix-exclusions.json` with the real reason; do not swap in a rewritten version that changes what the test validates +- **never replace upstream test source files** — if an os-test `.c` file fails due to a platform difference (e.g. `sizeof(long)`), exclude it via `os-test-exclusions.json` with the real reason; do not swap in a rewritten version that changes what the test validates - **kernel behavior belongs in the kernel, not the test runner** — if a test requires runtime state (POSIX directories like `/tmp`, `/usr`, device nodes, etc.), implement it in the kernel/device-layer so all users get it; the test runner should not create kernel state that real users won't have - **no suite-specific VFS special-casing** — the test runner must not branch on suite name to inject different filesystem state; if a test needs files to exist, either the kernel should provide them or the test should be excluded - **categorize exclusions honestly** — if a failure is fixable with a patch or build flag, it's `implementation-gap`, not `wasm-limitation`; reserve `wasm-limitation` for things genuinely impossible in wasm32-wasip1 (no 80-bit long double, no fork, no mmap) @@ -90,7 +90,7 @@ ## GitHub Issues - when fixing a bug or implementation gap tracked by a GitHub issue, close the issue in the same PR using `gh issue close --comment "Fixed in "` -- when removing a test from `posix-exclusions.json` because the fix landed, close the linked issue +- when removing a test from `os-test-exclusions.json` or `libc-test-exclusions.json` because the fix landed, close the linked issue - do not leave resolved issues open — verify with `gh issue view ` if unsure ## Tool Integration Policy diff --git a/docs-internal/posix-gaps-audit.md b/docs-internal/posix-gaps-audit.md new file mode 100644 index 00000000..38767fbb --- /dev/null +++ b/docs-internal/posix-gaps-audit.md @@ -0,0 +1,85 @@ +# POSIX Implementation Gaps Audit + +> Adversarial review of the WasmVM POSIX implementation. Goal: identify what breaks when real software runs unmodified. +> +> os-test conformance: 3347/3350 (99.9%) — but os-test only covers C library functions, not kernel/runtime behavior. +> +> Each claim was verified by independent adversarial agents reading the actual source code. + +## WILL_BREAK — Real software fails + +All 11 claims verified TRUE against source code. + +| # | Issue | What breaks | Where | Verified | +|---|-------|-------------|-------|----------| +| 1 | **No server sockets** (bind/listen/accept) | nginx, Express, Redis, Postgres, any daemon | wasi-ext has no bind/listen/accept imports; 0008-sockets.patch is client-only | TRUE | +| 2 | **No Unix domain sockets** (AF_UNIX) | docker.sock, ssh-agent, systemd socket activation | net_connect takes "host:port" string, no AF_UNIX path | TRUE | +| 3 | **O_EXCL not checked in fdOpen** | Atomic lock file creation (SQLite, Make, pkg managers) | kernel.ts fdOpen only handles O_CREAT; O_EXCL stored but never checked | TRUE | +| 4 | **O_TRUNC not applied in kernel fdOpen** | Shell `>` redirect, log rotation, any "w" mode open | kernel.ts fdOpen never truncates; Node bridge does but WasmVM kernel doesn't | TRUE | +| 5 | **readdir missing "." and ".."** | tar, find, shell globbing, POSIX compliance | in-memory-fs.ts listDirEntries never synthesizes "." or ".." | TRUE | +| 6 | **Blocking flock() returns EAGAIN immediately** | File-based locks (databases, build tools) | file-lock.ts:61 comment: "Blocking not implemented — treat as EAGAIN" | TRUE | +| 7 | **Pipe write EAGAIN without O_NONBLOCK** | Large pipelines (`tar \| gzip \| aws s3 cp`) | pipe-manager.ts:107 throws EAGAIN on full buffer; no retry in fd_write handler | TRUE | +| 8 | **Unlink open file doesn't defer delete** | Temp file pattern (create, unlink, keep writing) | in-memory-fs.ts removeFile unconditionally deletes; no refcount check | TRUE | +| 9 | **No signal handlers** (sigaction/signal) | Servers, databases, graceful shutdown | No sigaction syscall; signals delivered by default actions only | TRUE | +| 10 | **WASM can't be interrupted mid-compute** | Ctrl+C during tight loops hangs | Tight loops bypass Atomics.wait; worker.terminate() is hard kill | TRUE | +| 11 | **All inodes are 0** | Hard link detection, backup tools, `find -inum` | in-memory-fs.ts lines 156,172,332 hardcode ino:0; no inode allocator | TRUE | + +## TOO_THIN — Works for simple cases, breaks complex ones + +Fact-check found 4 claims were FALSE or EXAGGERATED. + +| # | Issue | What breaks | Verified | Notes | +|---|-------|-------------|----------|-------| +| 12 | **O_NONBLOCK not settable via fcntl F_SETFL** | Async I/O, event loops | ~~TRUE~~ **FALSE** | WasmVM fcntl.c DOES implement F_SETFL via __wasi_fd_fdstat_set_flags; only Node kernel lacks it | +| 13 | **PIPE_BUF atomicity not guaranteed** | Concurrent pipe writers | ~~TRUE~~ **FALSE** | JS is single-threaded — all pipe writes are atomic by definition; no interleaving possible | +| 14 | **pread/pwrite load entire file into memory** | Large file random access | ~~TRUE~~ **FALSE** | It's an InMemoryFS — files are already in memory; pread just slices a view. This is by design, not a bug | +| 15 | **/dev/ptmx stub** (doesn't allocate real PTY) | `script`, PTY-allocation | ~~TRUE~~ **FALSE** | Real PtyManager exists with full master/slave pairs, line discipline, and signal delivery. /dev/ptmx device-layer entry is VFS stub but PTY allocation happens via kernel ioctl | +| 16 | **poll() timeout -1 capped to 30s** | Event loops expecting indefinite blocking | **TRUE** | driver.ts:1098 hardcodes `timeout < 0 ? 30000 : timeout` | +| 17 | **No UDP sockets** | ping, DNS raw queries, DHCP | **TRUE** | SOCK_DGRAM accepted but silently creates TCP socket stub | +| 18 | **Socket send/recv ignore flags** (MSG_PEEK, MSG_DONTWAIT) | Protocol libraries | **TRUE** | Flags passed via RPC but never used in driver handlers | +| 19 | **setsockopt not implemented** (SO_REUSEADDR, TCP_NODELAY) | Port reuse, latency tuning | **TRUE** | kernel-worker.ts returns ENOSYS unconditionally | +| 20 | **Hard links don't increment nlink** | `ls -l`, backup dedup tools | **TRUE** | hardLinks Map tracked but stat always returns nlink:1 | +| 21 | **pthread_cond/barrier/rwlock/once not patched** | Python GIL, any C++ std::thread code | **TRUE** | Only mutex/key/attr patched; cond/barrier/rwlock/once use unpatched musl stubs that assume futex | +| 22 | **No iconv()** | Character set conversion | ~~TRUE~~ **FALSE** | musl's iconv IS compiled into wasi-libc; iconv.h available; charset support limited but present | +| 23 | **Timezone limited to UTC** | Locale-aware time formatting | **TRUE** | musl timezone code ifdef'd out for WASI; no tzdata in VFS; localtime returns UTC | +| 24 | **fcntl cloexec tracking limited to 256 FDs** | Programs with many open files | **TRUE** | fcntl.c:32 MAX_FDS=256; FDs >= 256 get EBADF | + +## MISSING — Not implemented at all + +Fact-check found several claims EXAGGERATED — the C interfaces exist but fail at WASI syscall layer (ENOSYS). The distinction matters: programs that check for availability at link time will succeed; programs that check at runtime will get clean errors. + +| # | Issue | Verified | Notes | +|---|-------|----------|-------| +| 25 | No fork() | **EXAGGERATED** | fork() callable via musl but returns ENOSYS from WASI layer | +| 26 | No epoll | **EXAGGERATED** | epoll stubs exist in musl; fail with ENOSYS at SYS_epoll_create1 | +| 27 | No named pipes (mkfifo) | **EXAGGERATED** | mkfifo/mknod exist; fail with ENOSYS at SYS_mknodat | +| 28 | No shared memory | **EXAGGERATED** | shm_open/shmget exist; fail with ENOSYS at syscall layer | +| 29 | No /proc population | **TRUE** | /proc directory created but empty; no self/exe/cpuinfo | +| 30 | No mmap | **EXAGGERATED** | mmap available via `-lwasi-emulated-mman` emulation layer | + +## What works well + +- Text processing pipelines (grep, sed, awk, jq, sort) +- Shell scripting (bash 5.x compatible via brush-shell) +- Build systems (make with subcommand spawning) +- HTTP/HTTPS clients (curl, wget, git clone, npm install) +- Interactive terminals (real PTY with line discipline, Ctrl+C for children) +- File I/O basics (read, write, create, delete, rename) +- Cross-runtime process spawning (WasmVM <-> Node <-> Python) +- SQLite (single-threaded, file-based) +- iconv (character set conversion via musl) +- Full PTY allocation with master/slave pairs and signal delivery +- O_NONBLOCK settable via fcntl F_SETFL in WasmVM + +## Corrected honest claim + +> "POSIX shell scripts, text processing tools, and HTTP clients run unmodified. Server sockets (bind/listen/accept), custom signal handlers, and true multi-threading are not supported. Most POSIX C interfaces exist and link correctly but several kernel-level operations (O_EXCL, O_TRUNC, readdir ./.. , blocking flock, deferred unlink) are missing and will break programs that depend on them." + +## Severity summary + +- **11 confirmed WILL_BREAK** issues (all verified true) +- **9 confirmed TOO_THIN** issues (4 original claims debunked as false) +- **1 confirmed MISSING** issue (5 original claims were exaggerated — interfaces exist, ENOSYS at runtime) +- **4 claims were FALSE** (O_NONBLOCK works, PIPE_BUF atomic in JS, pread fine for InMemoryFS, real PTY exists) +- **1 claim was FALSE** (iconv exists in musl) +- **5 claims were EXAGGERATED** (fork/epoll/mkfifo/shm/mmap exist as C stubs returning ENOSYS) diff --git a/docs/docs.json b/docs/docs.json index c312613b..9fb252e8 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -125,7 +125,8 @@ "group": "Reference", "pages": [ "posix-compatibility", - "posix-conformance-report", + "os-test-conformance-report", + "libc-test-conformance-report", "nodejs-conformance-report", "python-compatibility" ] diff --git a/docs/libc-test-conformance-report.mdx b/docs/libc-test-conformance-report.mdx new file mode 100644 index 00000000..64b70c87 --- /dev/null +++ b/docs/libc-test-conformance-report.mdx @@ -0,0 +1,60 @@ +--- +title: libc-test Conformance Report +description: musl libc-test kernel behavior conformance results for WasmVM. +icon: "chart-bar" +--- + +{/* AUTO-GENERATED — do not edit. Run scripts/generate-libc-test-report.ts */} + +## Summary + +musl libc-test tests actual kernel behavior — file locking, socket operations, stat edge cases, +and process management. Unlike os-test (which tests libc function correctness), these tests +exercise the runtime and kernel layer. + +| Metric | Value | +| --- | --- | +| libc-test version | master | +| Total tests | 75 | +| Passing | 69 (92.0%) | +| Expected fail | 6 | +| Skip | 0 | +| Native verified | 68 of 69 passing tests verified against native output (98.6%) | +| Last updated | 2026-03-26 | + +## Per-Suite Results + +| Suite | Total | Pass | Fail | Skip | Pass Rate | +| --- | --- | --- | --- | --- | --- | +| functional | 41 | 37 | 4 | 0 | 90.2% | +| regression | 34 | 32 | 2 | 0 | 94.1% | +| **Total** | **75** | **69** | **6** | **0** | **100.0%** | + +## Exclusions by Category + +### WASM Limitations (4 entries) + +Features impossible in wasm32-wasip1. + +| Test | Reason | Issue | +| --- | --- | --- | +| `functional/dlopen_dso` | dlopen/dlsym not available in wasm32-wasip1 — no dynamic linking support | [#48](https://github.com/rivet-dev/secure-exec/issues/48) | +| `functional/tls_align_dso` | Thread-local storage with dynamic shared objects requires dlopen — not available in WASM | [#48](https://github.com/rivet-dev/secure-exec/issues/48) | +| `functional/tls_init_dso` | Thread-local storage initialization with DSOs requires dlopen — not available in WASM | [#48](https://github.com/rivet-dev/secure-exec/issues/48) | +| `regression/tls_get_new-dtv_dso` | TLS dynamic thread vector with DSOs requires dlopen — not available in WASM | [#48](https://github.com/rivet-dev/secure-exec/issues/48) | + +### WASI Gaps (1 entry) + +WASI Preview 1 lacks the required syscall. + +| Test | Reason | Issue | +| --- | --- | --- | +| `regression/statvfs` | statvfs/fstatvfs not part of WASI — no filesystem statistics interface | [#48](https://github.com/rivet-dev/secure-exec/issues/48) | + +### Implementation Gaps (1 entry) + +Features we should support but don't yet. Each has a tracking issue. + +| Test | Reason | Issue | +| --- | --- | --- | +| `functional/strptime` | strptime fails on timezone-related format specifiers (%Z, %z) — musl timezone code is ifdef'd out for WASI | [#48](https://github.com/rivet-dev/secure-exec/issues/48) | diff --git a/docs/posix-conformance-report.mdx b/docs/os-test-conformance-report.mdx similarity index 75% rename from docs/posix-conformance-report.mdx rename to docs/os-test-conformance-report.mdx index ed708103..af700f1c 100644 --- a/docs/posix-conformance-report.mdx +++ b/docs/os-test-conformance-report.mdx @@ -1,10 +1,10 @@ --- -title: POSIX Conformance Report +title: os-test Conformance Report description: os-test POSIX.1-2024 conformance results for WasmVM. icon: "chart-bar" --- -{/* AUTO-GENERATED — do not edit. Run scripts/generate-posix-report.ts */} +{/* AUTO-GENERATED — do not edit. Run scripts/generate-os-test-report.ts */} ## Summary @@ -15,8 +15,8 @@ icon: "chart-bar" | Passing | 3347 (99.9%) | | Expected fail | 3 | | Skip | 0 | -| Native parity | 98.4% | -| Last updated | 2026-03-23 | +| Native verified | 3295 of 3347 passing tests verified against native output (98.4%) | +| Last updated | 2026-03-24 | ## Per-Suite Results @@ -47,5 +47,5 @@ WASI Preview 1 lacks the required syscall. | Test | Reason | Issue | | --- | --- | --- | -| `basic/sys_statvfs/fstatvfs` | fstatvfs() not part of WASI — no filesystem statistics interface | [#34](https://github.com/rivet-dev/secure-exec/issues/34) | -| `basic/sys_statvfs/statvfs` | statvfs() not part of WASI — no filesystem statistics interface | [#34](https://github.com/rivet-dev/secure-exec/issues/34) | +| `basic/sys_statvfs/fstatvfs` | fstatvfs() not part of WASI — no filesystem statistics interface | [#48](https://github.com/rivet-dev/secure-exec/issues/48) | +| `basic/sys_statvfs/statvfs` | statvfs() not part of WASI — no filesystem statistics interface | [#48](https://github.com/rivet-dev/secure-exec/issues/48) | diff --git a/docs/posix-compatibility.md b/docs/posix-compatibility.md index 169df4c3..c67eb312 100644 --- a/docs/posix-compatibility.md +++ b/docs/posix-compatibility.md @@ -2,7 +2,7 @@ > **This is a living document.** Update it when kernel, WasmVM, Node bridge, or Python bridge behavior changes for any POSIX-relevant feature. -> **Looking for automated test results?** See the [POSIX Conformance Report](posix-conformance-report.mdx) for os-test suite results with per-suite pass rates and exclusion details. +> **Looking for automated test results?** See the [os-test Conformance Report](os-test-conformance-report.mdx) for libc function correctness and the [libc-test Conformance Report](libc-test-conformance-report.mdx) for kernel behavior testing. This document tracks how closely the secure-exec kernel, runtimes, and bridges conform to POSIX and Linux behavior. The goal is full POSIX compliance 1:1 — every syscall, signal, and shell behavior should match a real Linux system unless an architectural constraint makes it impossible. diff --git a/libc-test-conformance-report.json b/libc-test-conformance-report.json new file mode 100644 index 00000000..d11c4c1b --- /dev/null +++ b/libc-test-conformance-report.json @@ -0,0 +1,552 @@ +{ + "libcTestVersion": "master", + "timestamp": "2026-03-26T07:23:29.899Z", + "total": 75, + "pass": 69, + "fail": 6, + "skip": 0, + "passRate": "92.0%", + "nativeVerified": 68, + "suites": { + "functional": { + "total": 41, + "pass": 37, + "fail": 4, + "skip": 0 + }, + "regression": { + "total": 34, + "pass": 32, + "fail": 2, + "skip": 0 + } + }, + "tests": [ + { + "name": "functional/argv", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/basename", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/clocale_mbfuncs", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 1 + }, + { + "name": "functional/clock_gettime", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/crypt", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 1 + }, + { + "name": "functional/dirname", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/dlopen_dso", + "suite": "functional", + "status": "fail", + "wasmExitCode": 1, + "wasmStderr": "unreachable\nunreachable\n" + }, + { + "name": "functional/env", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/fnmatch", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 1 + }, + { + "name": "functional/iconv_open", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/mbc", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/memstream", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/random", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/search_hsearch", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/search_insque", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/search_lsearch", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/search_tsearch", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/snprintf", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/sscanf", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 1 + }, + { + "name": "functional/strftime", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 1 + }, + { + "name": "functional/string", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0 + }, + { + "name": "functional/string_memcpy", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/string_memmem", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/string_memset", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/string_strchr", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/string_strcspn", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/string_strstr", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/strptime", + "suite": "functional", + "status": "fail", + "wasmExitCode": 1, + "nativeExitCode": 0, + "wasmStdout": "libc-test/src/functional/strptime.c:23: \"%F\": failed to parse \"1856-07-10\"\nlibc-test/src/functional/strptime.c:23: \"%F\": failed to parse \"1856-07-10\"\nlibc-test/src/functional/strptime.c:23: \"%s\": failed to parse \"683078400\"\nlibc-test/src/functional/strptime.c:23: \"%s\": failed to parse \"683078400\"\nlibc-test/src/functional/strptime.c:47: \"%z\": failed to parse \"+0200\"\nlibc-test/src/functional/strptime.c:47: \"%z\": failed to parse \"+0200\"\nlibc-test/src/functional/strptime.c:47: \"%z\": failed to parse \"-0530\"\nlibc-test/src/functional/strptime.c:47: \"%z\": failed to parse \"-0530\"\nlibc-test/src/functional/strptime.c:47: \"%z\": failed to parse \"-06\"\nlibc-test/src/functional/strptime.c:47: \"%z\": failed to parse \"-06\"\n" + }, + { + "name": "functional/strtod", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/strtod_long", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/strtod_simple", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/strtof", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/strtol", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 1 + }, + { + "name": "functional/strtold", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/swprintf", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 1 + }, + { + "name": "functional/tgmath", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/tls_align_dso", + "suite": "functional", + "status": "fail", + "wasmExitCode": 1, + "wasmStderr": "unreachable\nunreachable\n" + }, + { + "name": "functional/tls_init_dso", + "suite": "functional", + "status": "fail", + "wasmExitCode": 1, + "wasmStderr": "unreachable\nunreachable\n" + }, + { + "name": "functional/udiv", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/wcsstr", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "functional/wcstol", + "suite": "functional", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 1 + }, + { + "name": "regression/fgets-eof", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/fpclassify-invalid-ld80", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 1 + }, + { + "name": "regression/iconv-roundtrips", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/inet_ntop-v4mapped", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/inet_pton-empty-last-field", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/iswspace-null", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/lrand48-signextend", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/malloc-0", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/mbsrtowcs-overflow", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/memmem-oob", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/memmem-oob-read", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/printf-1e9-oob", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/printf-fmt-g-round", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/printf-fmt-g-zeros", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/printf-fmt-n", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/putenv-doublefree", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/regex-backref-0", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/regex-bracket-icase", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/regex-ere-backref", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 1 + }, + { + "name": "regression/regex-escaped-high-byte", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 1 + }, + { + "name": "regression/regex-negated-range", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/regexec-nosub", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/rewind-clear-error", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/scanf-bytes-consumed", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/scanf-match-literal-eof", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/scanf-nullbyte-char", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/setvbuf-unget", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 1 + }, + { + "name": "regression/sscanf-eof", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/statvfs", + "suite": "regression", + "status": "fail", + "wasmExitCode": 1, + "nativeExitCode": 0, + "wasmStdout": "libc-test/src/regression/statvfs.c:14: statvfs(\"/\") failed: Function not implemented\nlibc-test/src/regression/statvfs.c:14: statvfs(\"/\") failed: Function not implemented\nlibc-test/src/regression/statvfs.c:16: / has bogus f_bsize: 0\nlibc-test/src/regression/statvfs.c:16: / has bogus f_bsize: 0\nlibc-test/src/regression/statvfs.c:18: / has 0 blocks\nlibc-test/src/regression/statvfs.c:18: / has 0 blocks\nlibc-test/src/regression/statvfs.c:26: / has 0 file nodes\nlibc-test/src/regression/statvfs.c:26: / has 0 file nodes\nlibc-test/src/regression/statvfs.c:34: / has bogus f_namemax: 0\nlibc-test/src/regression/statvfs.c:34: / has bogus f_namemax: 0\n" + }, + { + "name": "regression/strverscmp", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/tls_get_new-dtv_dso", + "suite": "regression", + "status": "fail", + "wasmExitCode": 1, + "wasmStderr": "unreachable\nunreachable\n" + }, + { + "name": "regression/uselocale-0", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/wcsncpy-read-overflow", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + }, + { + "name": "regression/wcsstr-false-negative", + "suite": "regression", + "status": "pass", + "wasmExitCode": 0, + "nativeExitCode": 0 + } + ] +} \ No newline at end of file diff --git a/native/wasmvm/c/.gitignore b/native/wasmvm/c/.gitignore index d5c4b83a..d7d082bb 100644 --- a/native/wasmvm/c/.gitignore +++ b/native/wasmvm/c/.gitignore @@ -13,5 +13,8 @@ # os-test conformance suite (downloaded at build time via make fetch-os-test) /os-test/ +# musl libc-test suite (downloaded at build time via make fetch-libc-test) +/libc-test/ + # Compiled WASM binaries (output of make programs) /build/ diff --git a/native/wasmvm/c/Makefile b/native/wasmvm/c/Makefile index ef167bd7..5117c248 100644 --- a/native/wasmvm/c/Makefile +++ b/native/wasmvm/c/Makefile @@ -272,6 +272,100 @@ os-test-native: fetch-os-test echo "Output: $(OS_TEST_NATIVE_BUILD)/"; \ echo "=== Build complete ===" +# --- musl libc-test conformance suite (downloaded at build time) --- +# MIT-licensed test suite from musl (Bytecode Alliance mirror) + +LIBC_TEST_VERSION := master +LIBC_TEST_URL := https://github.com/bytecodealliance/libc-test/archive/refs/heads/$(LIBC_TEST_VERSION).tar.gz +LIBC_TEST_DIR := libc-test + +.PHONY: fetch-libc-test clean-libc-test + +fetch-libc-test: $(LIBC_TEST_DIR)/src +$(LIBC_TEST_DIR)/src: + @echo "Fetching musl libc-test $(LIBC_TEST_VERSION)..." + @mkdir -p $(LIBS_CACHE) + @curl -fSL "$(LIBC_TEST_URL)" -o "$(LIBS_CACHE)/libc-test.tar.gz" + @mkdir -p $(LIBC_TEST_DIR) + @tar -xzf "$(LIBS_CACHE)/libc-test.tar.gz" --strip-components=1 -C $(LIBC_TEST_DIR) + @echo "libc-test $(LIBC_TEST_VERSION) extracted to $(LIBC_TEST_DIR)/" + @echo " src/ : $$(find $(LIBC_TEST_DIR)/src -name '*.c' 2>/dev/null | wc -l) test programs" + +clean-libc-test: + rm -rf $(LIBC_TEST_DIR) $(LIBS_CACHE)/libc-test.tar.gz + +# --- musl libc-test build targets --- +# Compile functional/ and regression/ tests to WASM and native binaries. +# Tests requiring fork/exec/signals/pthreads will fail to compile for WASM — that's expected. +# The test.h framework (src/common/) is compiled into a static library first. + +LIBC_TEST_BUILD := $(BUILD_DIR)/libc-test +LIBC_TEST_NATIVE_BUILD := $(NATIVE_DIR)/libc-test +LIBC_TEST_WASM_CFLAGS := --target=wasm32-wasip1 --sysroot=$(SYSROOT) -O0 \ + -D_GNU_SOURCE -D_BSD_SOURCE -D_ALL_SOURCE -D_DEFAULT_SOURCE \ + -I$(LIBC_TEST_DIR)/src/common +LIBC_TEST_WASM_LDFLAGS := -lc-printscan-long-double -lm +LIBC_TEST_NATIVE_CFLAGS := -O0 -g \ + -D_GNU_SOURCE -D_BSD_SOURCE -D_ALL_SOURCE -D_DEFAULT_SOURCE \ + -I$(LIBC_TEST_DIR)/src/common +LIBC_TEST_NATIVE_LDFLAGS := -lm +ifeq ($(UNAME_S),Linux) + LIBC_TEST_NATIVE_LDFLAGS += -lpthread -lrt -lcrypt -lresolv -lutil -ldl +endif +ifeq ($(UNAME_S),Darwin) + LIBC_TEST_NATIVE_LDFLAGS += -lpthread +endif + +# Test framework sources compiled alongside each test +LIBC_TEST_COMMON_SRCS := $(LIBC_TEST_DIR)/src/common/print.c + +.PHONY: libc-test libc-test-native + +libc-test: wasi-sdk fetch-libc-test + @total=0; pass=0; fail=0; \ + for src in $$(find $(LIBC_TEST_DIR)/src/functional $(LIBC_TEST_DIR)/src/regression -name '*.c' 2>/dev/null | sort); do \ + rel=$${src#$(LIBC_TEST_DIR)/src/}; \ + name=$${rel%.c}; \ + out=$(LIBC_TEST_BUILD)/$$name; \ + mkdir -p "$$(dirname "$$out")"; \ + total=$$((total + 1)); \ + if $(CC) $(LIBC_TEST_WASM_CFLAGS) -o "$$out" "$$src" $(LIBC_TEST_COMMON_SRCS) $(LIBC_TEST_WASM_LDFLAGS) 2>/dev/null; then \ + pass=$$((pass + 1)); \ + else \ + rm -f "$$out"; \ + fail=$$((fail + 1)); \ + fi; \ + done; \ + echo ""; \ + echo "=== libc-test WASM Build Report ==="; \ + echo "Total: $$total"; \ + echo "Compiled: $$pass"; \ + echo "Failed: $$fail"; \ + echo "Output: $(LIBC_TEST_BUILD)/"; \ + echo "=== Build complete ===" + +libc-test-native: fetch-libc-test + @total=0; pass=0; fail=0; \ + for src in $$(find $(LIBC_TEST_DIR)/src/functional $(LIBC_TEST_DIR)/src/regression -name '*.c' 2>/dev/null | sort); do \ + rel=$${src#$(LIBC_TEST_DIR)/src/}; \ + out=$(LIBC_TEST_NATIVE_BUILD)/$${rel%.c}; \ + mkdir -p "$$(dirname "$$out")"; \ + total=$$((total + 1)); \ + if $(NATIVE_CC) $(LIBC_TEST_NATIVE_CFLAGS) -o "$$out" "$$src" $(LIBC_TEST_COMMON_SRCS) $(LIBC_TEST_NATIVE_LDFLAGS) 2>/dev/null; then \ + pass=$$((pass + 1)); \ + else \ + rm -f "$$out"; \ + fail=$$((fail + 1)); \ + fi; \ + done; \ + echo ""; \ + echo "=== libc-test Native Build Report ==="; \ + echo "Total: $$total"; \ + echo "Compiled: $$pass"; \ + echo "Failed: $$fail"; \ + echo "Output: $(LIBC_TEST_NATIVE_BUILD)/"; \ + echo "=== Build complete ===" + # --- Patched sysroot (delegates to patch-wasi-libc.sh) --- sysroot: wasi-sdk diff --git a/posix-conformance-report.json b/os-test-conformance-report.json similarity index 100% rename from posix-conformance-report.json rename to os-test-conformance-report.json diff --git a/packages/wasmvm/test/libc-test-conformance.test.ts b/packages/wasmvm/test/libc-test-conformance.test.ts new file mode 100644 index 00000000..89dcaf7b --- /dev/null +++ b/packages/wasmvm/test/libc-test-conformance.test.ts @@ -0,0 +1,383 @@ +/** + * Kernel behavior conformance tests — musl libc-test suite + * + * Discovers all compiled libc-test WASM binaries (functional/ and regression/), + * checks them against the exclusion list, and runs everything not excluded + * through the WasmVM kernel. Native binaries are run for parity comparison. + * + * Unlike os-test (which tests libc function correctness), libc-test exercises + * kernel-level behavior: file locking, socket operations, stat edge cases, + * signal delivery, and process management. + * + * Tests skip gracefully when WASM binaries are not built. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { createWasmVmRuntime } from '../src/driver.ts'; +import { createKernel, createInMemoryFileSystem } from '@secure-exec/core'; +import type { Kernel } from '@secure-exec/core'; +import { + existsSync, + readdirSync, + statSync, + symlinkSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { spawn } from 'node:child_process'; +import { resolve, dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { tmpdir } from 'node:os'; +import type { ExclusionEntry } from '../../../scripts/conformance-exclusion-schema.js'; +import exclusionsData from './libc-test-exclusions.json'; + +// ── Paths ────────────────────────────────────────────────────────────── + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const COMMANDS_DIR = resolve( + __dirname, + '../../../native/wasmvm/target/wasm32-wasip1/release/commands', +); +const C_BUILD_DIR = resolve(__dirname, '../../../native/wasmvm/c/build'); +const LIBC_TEST_WASM_DIR = join(C_BUILD_DIR, 'libc-test'); +const LIBC_TEST_NATIVE_DIR = join(C_BUILD_DIR, 'native', 'libc-test'); +const REPORT_PATH = resolve(__dirname, '../../../libc-test-conformance-report.json'); + +const TEST_TIMEOUT_MS = 30_000; +const NATIVE_TIMEOUT_MS = 25_000; + +const hasWasmBinaries = existsSync(COMMANDS_DIR); +const hasLibcTestWasm = existsSync(LIBC_TEST_WASM_DIR); + +// ── Skip guard ───────────────────────────────────────────────────────── + +function skipReason(): string | false { + if (!hasWasmBinaries) return 'WASM runtime binaries not built (run make wasm in native/wasmvm/)'; + if (!hasLibcTestWasm) return 'libc-test WASM binaries not built (run make -C native/wasmvm/c libc-test)'; + return false; +} + +// ── Test discovery ───────────────────────────────────────────────────── + +function discoverTests(dir: string, prefix = ''): string[] { + if (!existsSync(dir)) return []; + const results: string[] = []; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isDirectory()) { + results.push(...discoverTests(join(dir, entry.name), rel)); + } else { + results.push(rel); + } + } + return results.sort(); +} + +// ── Native binary runner ─────────────────────────────────────────────── + +function runNative( + path: string, + cwd?: string, +): Promise<{ exitCode: number; stdout: string; stderr: string }> { + return new Promise((res) => { + const proc = spawn(path, [], { stdio: ['pipe', 'pipe', 'pipe'], cwd }); + let stdout = ''; + let stderr = ''; + const timer = setTimeout(() => { + proc.kill('SIGKILL'); + }, NATIVE_TIMEOUT_MS); + + proc.stdout.on('data', (d: Buffer) => { stdout += d.toString(); }); + proc.stderr.on('data', (d: Buffer) => { stderr += d.toString(); }); + proc.stdin.end(); + proc.on('close', (code) => { + clearTimeout(timer); + res({ exitCode: code ?? 1, stdout, stderr }); + }); + }); +} + +// ── Flat symlink directory for command resolution ────────────────────── + +function toFlatName(testName: string): string { + return 'libc-test--' + testName.replaceAll('/', '--'); +} + +const allTests = hasLibcTestWasm ? discoverTests(LIBC_TEST_WASM_DIR) : []; +let FLAT_CMD_DIR: string | undefined; + +if (allTests.length > 0) { + FLAT_CMD_DIR = mkdtempSync(join(tmpdir(), 'libc-test-')); + for (const test of allTests) { + symlinkSync(join(LIBC_TEST_WASM_DIR, test), join(FLAT_CMD_DIR, toFlatName(test))); + } +} + +// ── Exclusion map ────────────────────────────────────────────────────── + +const exclusions = exclusionsData.exclusions as Record; + +// ── Group by suite ───────────────────────────────────────────────────── + +const bySuite = new Map(); +for (const test of allTests) { + const suite = test.includes('/') ? test.split('/')[0] : 'root'; + if (!bySuite.has(suite)) bySuite.set(suite, []); + bySuite.get(suite)!.push(test); +} + +// ── Result tracking ──────────────────────────────────────────────────── + +interface TestResult { + name: string; + suite: string; + status: 'pass' | 'fail' | 'skip'; + wasmExitCode?: number; + nativeExitCode?: number; + wasmStdout?: string; + nativeStdout?: string; + wasmStderr?: string; + nativeStderr?: string; + error?: string; +} + +const testResults: TestResult[] = []; + +// ── Report generation ────────────────────────────────────────────────── + +function writeConformanceReport(results: TestResult[]): void { + const suites: Record = {}; + for (const r of results) { + if (!suites[r.suite]) suites[r.suite] = { total: 0, pass: 0, fail: 0, skip: 0 }; + suites[r.suite].total++; + suites[r.suite][r.status]++; + } + + const total = results.length; + const pass = results.filter((r) => r.status === 'pass').length; + const fail = results.filter((r) => r.status === 'fail').length; + const skip = results.filter((r) => r.status === 'skip').length; + const passRate = total - skip > 0 + ? ((pass / (total - skip)) * 100).toFixed(1) + : '0.0'; + + const nativeVerifiedCount = results.filter( + (r) => r.status === 'pass' && r.nativeExitCode !== undefined, + ).length; + + const report = { + libcTestVersion: exclusionsData.libcTestVersion, + timestamp: new Date().toISOString(), + total, + pass, + fail, + skip, + passRate: `${passRate}%`, + nativeVerified: nativeVerifiedCount, + suites, + tests: results, + }; + + writeFileSync(REPORT_PATH, JSON.stringify(report, null, 2)); +} + +function printSummary(results: TestResult[]): void { + const suites: Record = {}; + for (const r of results) { + if (!suites[r.suite]) suites[r.suite] = { total: 0, pass: 0, fail: 0, skip: 0 }; + suites[r.suite].total++; + suites[r.suite][r.status]++; + } + + const total = results.length; + const pass = results.filter((r) => r.status === 'pass').length; + const fail = results.filter((r) => r.status === 'fail').length; + const skip = results.filter((r) => r.status === 'skip').length; + const mustPass = total - skip; + const passRate = mustPass > 0 ? ((pass / mustPass) * 100).toFixed(1) : '—'; + + console.log(''); + console.log(`libc-test Conformance Summary (musl libc-test ${exclusionsData.libcTestVersion})`); + console.log('─'.repeat(60)); + console.log( + 'Suite'.padEnd(20) + + 'Total'.padStart(8) + + 'Pass'.padStart(8) + + 'Fail'.padStart(8) + + 'Skip'.padStart(8) + + 'Rate'.padStart(10), + ); + + for (const [name, s] of Object.entries(suites).sort(([a], [b]) => a.localeCompare(b))) { + const runnable = s.total - s.skip; + const rate = runnable > 0 + ? ((s.pass / runnable) * 100).toFixed(1) + '%' + : '—'; + console.log( + name.padEnd(20) + + String(s.total).padStart(8) + + String(s.pass).padStart(8) + + String(s.fail).padStart(8) + + String(s.skip).padStart(8) + + rate.padStart(10), + ); + } + + console.log('─'.repeat(60)); + console.log( + 'TOTAL'.padEnd(20) + + String(total).padStart(8) + + String(pass).padStart(8) + + String(fail).padStart(8) + + String(skip).padStart(8) + + (passRate + (passRate !== '—' ? '%' : '')).padStart(10), + ); + + console.log(`Expected fail: ${fail}`); + console.log(`Must-pass: ${mustPass - fail} (${pass} passing)`); + console.log(''); +} + +// ── Test suite ───────────────────────────────────────────────────────── + +describe.skipIf(skipReason())('libc-test conformance (musl)', () => { + afterAll(() => { + if (testResults.length > 0) { + writeConformanceReport(testResults); + printSummary(testResults); + } + if (FLAT_CMD_DIR) { + try { rmSync(FLAT_CMD_DIR, { recursive: true, force: true }); } catch { /* ignore */ } + } + }); + + for (const [suite, tests] of bySuite) { + describe(`libc-test/${suite}`, () => { + let kernel: Kernel; + + const nativeSuiteCwd = join(LIBC_TEST_NATIVE_DIR, suite); + + beforeAll(async () => { + // libc-test functional tests mostly operate on files they create + // themselves (mkstemp, etc.), so a minimal VFS is sufficient + const filesystem = createInMemoryFileSystem(); + // Create /tmp for tests that use mkstemp/mkdtemp + await filesystem.mkdir('/tmp'); + kernel = createKernel({ filesystem, cwd: '/tmp' }); + await kernel.mount( + createWasmVmRuntime({ commandDirs: [FLAT_CMD_DIR!, COMMANDS_DIR] }), + ); + }); + + afterAll(async () => { + await kernel?.dispose(); + }); + + for (const testName of tests) { + const exclusion = exclusions[testName]; + + if (exclusion?.expected === 'skip') { + it.skip(`${testName} — ${exclusion.reason}`, () => {}); + testResults.push({ name: testName, suite, status: 'skip' }); + continue; + } + + it(testName, async () => { + const flatName = toFlatName(testName); + + // Run natively (if binary exists) + const nativePath = join(LIBC_TEST_NATIVE_DIR, testName); + const nativeResult = existsSync(nativePath) + ? await runNative(nativePath, existsSync(nativeSuiteCwd) ? nativeSuiteCwd : undefined) + : null; + + // Run in WASM via kernel.spawn() + const stdoutChunks: Uint8Array[] = []; + const stderrChunks: Uint8Array[] = []; + const proc = kernel.spawn(flatName, [], { + onStdout: (d) => stdoutChunks.push(d), + onStderr: (d) => stderrChunks.push(d), + timeout: NATIVE_TIMEOUT_MS, + }); + proc.closeStdin(); + const wasmExitCode = await proc.wait(); + const wasmStdout = Buffer.concat(stdoutChunks).toString(); + const wasmStderr = Buffer.concat(stderrChunks).toString(); + const wasmResult = { exitCode: wasmExitCode, stdout: wasmStdout, stderr: wasmStderr }; + + if (exclusion?.expected === 'fail') { + // Known failure — must still fail + const exitOk = wasmResult.exitCode === 0; + const parityOk = !nativeResult || nativeResult.exitCode !== 0 || + wasmResult.stdout.trim() === nativeResult.stdout.trim(); + const nativeParityPass = !!nativeResult && + nativeResult.exitCode !== 0 && + wasmResult.exitCode === nativeResult.exitCode && + wasmResult.stdout.trim() === nativeResult.stdout.trim(); + + if ((exitOk && parityOk) || nativeParityPass) { + testResults.push({ + name: testName, suite, status: 'pass', + wasmExitCode: 0, nativeExitCode: nativeResult?.exitCode, + wasmStdout: wasmStdout || undefined, + wasmStderr: wasmStderr || undefined, + }); + throw new Error( + `${testName} is excluded as "fail" but now passes! ` + + 'Remove it from libc-test-exclusions.json to lock in this fix.', + ); + } + testResults.push({ + name: testName, suite, status: 'fail', + wasmExitCode: wasmResult.exitCode, + nativeExitCode: nativeResult?.exitCode, + wasmStdout: wasmStdout || undefined, + wasmStderr: wasmStderr || undefined, + }); + } else { + // Not excluded — must pass (or match native failure exactly) + try { + // libc-test pass = exit 0, no stdout output + // libc-test fail = exit 1, error messages on stdout + if (nativeResult && nativeResult.exitCode !== 0 && + wasmResult.exitCode === nativeResult.exitCode && + wasmResult.stdout.trim() === nativeResult.stdout.trim()) { + // Both fail identically — native parity + testResults.push({ + name: testName, suite, status: 'pass', + wasmExitCode: wasmResult.exitCode, + nativeExitCode: nativeResult.exitCode, + }); + } else { + expect(wasmResult.exitCode).toBe(0); + + // Native parity: if native passes, output should match + // (libc-test passes produce no stdout) + if (nativeResult && nativeResult.exitCode === 0) { + expect(wasmResult.stdout.trim()).toBe(nativeResult.stdout.trim()); + } + + testResults.push({ + name: testName, suite, status: 'pass', + wasmExitCode: wasmResult.exitCode, + nativeExitCode: nativeResult?.exitCode, + }); + } + } catch (err) { + testResults.push({ + name: testName, suite, status: 'fail', + wasmExitCode: wasmResult.exitCode, + nativeExitCode: nativeResult?.exitCode, + wasmStdout: wasmStdout || undefined, + wasmStderr: wasmStderr || undefined, + error: (err as Error).message, + }); + throw err; + } + } + }, TEST_TIMEOUT_MS); + } + }); + } +}); diff --git a/packages/wasmvm/test/libc-test-exclusions.json b/packages/wasmvm/test/libc-test-exclusions.json new file mode 100644 index 00000000..5231b7d1 --- /dev/null +++ b/packages/wasmvm/test/libc-test-exclusions.json @@ -0,0 +1,43 @@ +{ + "libcTestVersion": "master", + "sourceCommit": "master", + "lastUpdated": "2026-03-26", + "exclusions": { + "functional/dlopen_dso": { + "expected": "fail", + "category": "wasm-limitation", + "reason": "dlopen/dlsym not available in wasm32-wasip1 — no dynamic linking support", + "issue": "https://github.com/rivet-dev/secure-exec/issues/48" + }, + "functional/tls_align_dso": { + "expected": "fail", + "category": "wasm-limitation", + "reason": "Thread-local storage with dynamic shared objects requires dlopen — not available in WASM", + "issue": "https://github.com/rivet-dev/secure-exec/issues/48" + }, + "functional/tls_init_dso": { + "expected": "fail", + "category": "wasm-limitation", + "reason": "Thread-local storage initialization with DSOs requires dlopen — not available in WASM", + "issue": "https://github.com/rivet-dev/secure-exec/issues/48" + }, + "functional/strptime": { + "expected": "fail", + "category": "implementation-gap", + "reason": "strptime fails on timezone-related format specifiers (%Z, %z) — musl timezone code is ifdef'd out for WASI", + "issue": "https://github.com/rivet-dev/secure-exec/issues/48" + }, + "regression/statvfs": { + "expected": "fail", + "category": "wasi-gap", + "reason": "statvfs/fstatvfs not part of WASI — no filesystem statistics interface", + "issue": "https://github.com/rivet-dev/secure-exec/issues/48" + }, + "regression/tls_get_new-dtv_dso": { + "expected": "fail", + "category": "wasm-limitation", + "reason": "TLS dynamic thread vector with DSOs requires dlopen — not available in WASM", + "issue": "https://github.com/rivet-dev/secure-exec/issues/48" + } + } +} diff --git a/packages/wasmvm/test/posix-conformance.test.ts b/packages/wasmvm/test/os-test-conformance.test.ts similarity index 98% rename from packages/wasmvm/test/posix-conformance.test.ts rename to packages/wasmvm/test/os-test-conformance.test.ts index daa94fd6..e934cf5b 100644 --- a/packages/wasmvm/test/posix-conformance.test.ts +++ b/packages/wasmvm/test/os-test-conformance.test.ts @@ -25,8 +25,8 @@ import { spawn } from 'node:child_process'; import { resolve, dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { tmpdir } from 'node:os'; -import type { ExclusionEntry } from '../../../scripts/posix-exclusion-schema.js'; -import exclusionsData from './posix-exclusions.json'; +import type { ExclusionEntry } from '../../../scripts/conformance-exclusion-schema.js'; +import exclusionsData from './os-test-exclusions.json'; // ── Paths ────────────────────────────────────────────────────────────── @@ -40,7 +40,7 @@ const C_SRC_DIR = resolve(__dirname, '../../../native/wasmvm/c'); const OS_TEST_WASM_DIR = join(C_BUILD_DIR, 'os-test'); const OS_TEST_NATIVE_DIR = join(C_BUILD_DIR, 'native', 'os-test'); const OS_TEST_SRC_DIR = join(C_SRC_DIR, 'os-test'); -const REPORT_PATH = resolve(__dirname, '../../../posix-conformance-report.json'); +const REPORT_PATH = resolve(__dirname, '../../../os-test-conformance-report.json'); const TEST_TIMEOUT_MS = 30_000; const NATIVE_TIMEOUT_MS = 25_000; @@ -420,7 +420,7 @@ describe.skipIf(skipReason())('POSIX conformance (os-test)', () => { }); throw new Error( `${testName} is excluded as "fail" but now passes! ` + - 'Remove it from posix-exclusions.json to lock in this fix.', + 'Remove it from os-test-exclusions.json to lock in this fix.', ); } testResults.push({ diff --git a/packages/wasmvm/test/posix-exclusions.json b/packages/wasmvm/test/os-test-exclusions.json similarity index 100% rename from packages/wasmvm/test/posix-exclusions.json rename to packages/wasmvm/test/os-test-exclusions.json diff --git a/scripts/posix-exclusion-schema.ts b/scripts/conformance-exclusion-schema.ts similarity index 87% rename from scripts/posix-exclusion-schema.ts rename to scripts/conformance-exclusion-schema.ts index 64d32e6e..36b28063 100644 --- a/scripts/posix-exclusion-schema.ts +++ b/scripts/conformance-exclusion-schema.ts @@ -1,11 +1,10 @@ /** - * Shared schema for posix-exclusions.json. + * Shared schema for conformance exclusion files. * * Single source of truth for valid categories, expected values, * and the ExclusionEntry interface. Used by: - * - validate-posix-exclusions.ts - * - generate-posix-report.ts - * - posix-conformance.test.ts + * - os-test-conformance.test.ts / validate-os-test-exclusions.ts / generate-os-test-report.ts + * - libc-test-conformance.test.ts / validate-libc-test-exclusions.ts / generate-libc-test-report.ts */ export const VALID_EXPECTED = ['fail', 'skip'] as const; @@ -29,10 +28,10 @@ export interface ExclusionEntry { } export interface ExclusionsFile { - osTestVersion: string; sourceCommit: string; lastUpdated: string; exclusions: Record; + [key: string]: unknown; } /** Category metadata for report generation (ordered for display). */ diff --git a/scripts/generate-libc-test-report.ts b/scripts/generate-libc-test-report.ts new file mode 100644 index 00000000..a89dcaae --- /dev/null +++ b/scripts/generate-libc-test-report.ts @@ -0,0 +1,188 @@ +#!/usr/bin/env -S npx tsx +/** + * Generates docs/libc-test-conformance-report.mdx from test results and exclusion data. + * + * Usage: pnpm tsx scripts/generate-libc-test-report.ts + * --input libc-test-conformance-report.json + * --exclusions packages/wasmvm/test/libc-test-exclusions.json + * --output docs/libc-test-conformance-report.mdx + */ + +import { readFileSync, writeFileSync } from 'node:fs'; +import { resolve, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; +import { VALID_CATEGORIES, CATEGORY_META, CATEGORY_ORDER } from './conformance-exclusion-schema.js'; +import type { ExclusionEntry, ExclusionsFile } from './conformance-exclusion-schema.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +// ── CLI args ──────────────────────────────────────────────────────────── + +const { values } = parseArgs({ + options: { + input: { type: 'string', default: resolve(__dirname, '../libc-test-conformance-report.json') }, + exclusions: { type: 'string', default: resolve(__dirname, '../packages/wasmvm/test/libc-test-exclusions.json') }, + output: { type: 'string', default: resolve(__dirname, '../docs/libc-test-conformance-report.mdx') }, + }, +}); + +const inputPath = resolve(values.input!); +const exclusionsPath = resolve(values.exclusions!); +const outputPath = resolve(values.output!); + +// ── Load data ─────────────────────────────────────────────────────────── + +interface SuiteStats { + total: number; + pass: number; + fail: number; + skip: number; +} + +interface Report { + libcTestVersion: string; + timestamp: string; + total: number; + pass: number; + fail: number; + skip: number; + passRate: string; + nativeVerified: number; + suites: Record; +} + +const report: Report = JSON.parse(readFileSync(inputPath, 'utf-8')); +const exclusionsData: ExclusionsFile & { libcTestVersion: string } = JSON.parse(readFileSync(exclusionsPath, 'utf-8')); + +// ── Group exclusions by category ──────────────────────────────────────── + +interface GroupedExclusion { + key: string; + entry: ExclusionEntry; +} + +const validCats = new Set(VALID_CATEGORIES); +const byCategory = new Map(); +for (const [key, entry] of Object.entries(exclusionsData.exclusions)) { + const cat = entry.category; + if (!validCats.has(cat)) { + throw new Error(`[${key}] Unknown category "${cat}" — valid: ${VALID_CATEGORIES.join(', ')}`); + } + if (!byCategory.has(cat)) byCategory.set(cat, []); + byCategory.get(cat)!.push({ key, entry }); +} + +function formatExclusionKey(key: string): string { + return `\`${key}\``; +} + +// ── Build MDX ─────────────────────────────────────────────────────────── + +const lines: string[] = []; + +function line(s = '') { + lines.push(s); +} + +// Frontmatter +line('---'); +line('title: libc-test Conformance Report'); +line('description: musl libc-test kernel behavior conformance results for WasmVM.'); +line('icon: "chart-bar"'); +line('---'); +line(); +line('{/* AUTO-GENERATED — do not edit. Run scripts/generate-libc-test-report.ts */}'); +line(); + +// Summary table +const nativeVerifiedPct = report.pass > 0 ? ((report.nativeVerified / report.pass) * 100).toFixed(1) : '0'; +const nativeVerifiedLabel = `${report.nativeVerified} of ${report.pass} passing tests verified against native output (${nativeVerifiedPct}%)`; +const lastUpdated = report.timestamp ? report.timestamp.split('T')[0] : exclusionsData.lastUpdated; + +line('## Summary'); +line(); +line('musl libc-test tests actual kernel behavior — file locking, socket operations, stat edge cases,'); +line('and process management. Unlike os-test (which tests libc function correctness), these tests'); +line('exercise the runtime and kernel layer.'); +line(); +line('| Metric | Value |'); +line('| --- | --- |'); +line(`| libc-test version | ${report.libcTestVersion} |`); +line(`| Total tests | ${report.total} |`); +line(`| Passing | ${report.pass} (${report.passRate}) |`); +line(`| Expected fail | ${report.fail} |`); +line(`| Skip | ${report.skip} |`); +line(`| Native verified | ${nativeVerifiedLabel} |`); +line(`| Last updated | ${lastUpdated} |`); +line(); + +// Per-suite results table +line('## Per-Suite Results'); +line(); +line('| Suite | Total | Pass | Fail | Skip | Pass Rate |'); +line('| --- | --- | --- | --- | --- | --- |'); + +const sortedSuites = Object.entries(report.suites).sort(([a], [b]) => a.localeCompare(b)); +for (const [suite, stats] of sortedSuites) { + const runnable = stats.total - stats.skip; + const rate = runnable > 0 ? `${((stats.pass / runnable) * 100).toFixed(1)}%` : '—'; + line(`| ${suite} | ${stats.total} | ${stats.pass} | ${stats.fail} | ${stats.skip} | ${rate} |`); +} + +// Totals row +const runTotal = report.total - report.skip - report.fail; +const totalRate = runTotal > 0 ? `${((report.pass / runTotal) * 100).toFixed(1)}%` : '—'; +line(`| **Total** | **${report.total}** | **${report.pass}** | **${report.fail}** | **${report.skip}** | **${totalRate}** |`); +line(); + +// Exclusions by category +const hasExclusions = [...byCategory.values()].some((entries) => entries.length > 0); +if (hasExclusions) { + line('## Exclusions by Category'); + line(); + + for (const cat of CATEGORY_ORDER) { + const entries = byCategory.get(cat); + if (!entries || entries.length === 0) continue; + + const meta = CATEGORY_META[cat]; + const totalExcluded = entries.length; + + line(`### ${meta.title} (${totalExcluded} ${totalExcluded === 1 ? 'entry' : 'entries'})`); + line(); + line(meta.description); + line(); + + const hasIssues = entries.some((e) => e.entry.issue); + + if (hasIssues) { + line('| Test | Reason | Issue |'); + line('| --- | --- | --- |'); + for (const { key, entry } of entries) { + const issueLink = entry.issue + ? `[${entry.issue.replace('https://github.com/rivet-dev/secure-exec/issues/', '#')}](${entry.issue})` + : '—'; + line(`| ${formatExclusionKey(key)} | ${entry.reason} | ${issueLink} |`); + } + } else { + line('| Test | Reason |'); + line('| --- | --- |'); + for (const { key, entry } of entries) { + line(`| ${formatExclusionKey(key)} | ${entry.reason} |`); + } + } + line(); + } +} + +// ── Write output ──────────────────────────────────────────────────────── + +const mdx = lines.join('\n'); +writeFileSync(outputPath, mdx, 'utf-8'); + +console.log(`libc-test Conformance Report generated`); +console.log(` Input: ${inputPath}`); +console.log(` Exclusions: ${exclusionsPath}`); +console.log(` Output: ${outputPath}`); +console.log(` Summary: ${report.pass}/${report.total} passing (${report.passRate})`); diff --git a/scripts/generate-posix-report.ts b/scripts/generate-os-test-report.ts similarity index 88% rename from scripts/generate-posix-report.ts rename to scripts/generate-os-test-report.ts index ec815706..c3ac7764 100644 --- a/scripts/generate-posix-report.ts +++ b/scripts/generate-os-test-report.ts @@ -1,19 +1,19 @@ #!/usr/bin/env -S npx tsx /** - * Generates docs/posix-conformance-report.mdx from test results and exclusion data. + * Generates docs/os-test-conformance-report.mdx from test results and exclusion data. * - * Usage: pnpm tsx scripts/generate-posix-report.ts - * --input posix-conformance-report.json - * --exclusions packages/wasmvm/test/posix-exclusions.json - * --output docs/posix-conformance-report.mdx + * Usage: pnpm tsx scripts/generate-os-test-report.ts + * --input os-test-conformance-report.json + * --exclusions packages/wasmvm/test/os-test-exclusions.json + * --output docs/os-test-conformance-report.mdx */ import { readFileSync, writeFileSync } from 'node:fs'; import { resolve, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { parseArgs } from 'node:util'; -import { VALID_CATEGORIES, CATEGORY_META, CATEGORY_ORDER } from './posix-exclusion-schema.js'; -import type { ExclusionEntry, ExclusionsFile } from './posix-exclusion-schema.js'; +import { VALID_CATEGORIES, CATEGORY_META, CATEGORY_ORDER } from './conformance-exclusion-schema.js'; +import type { ExclusionEntry, ExclusionsFile } from './conformance-exclusion-schema.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -21,9 +21,9 @@ const __dirname = dirname(fileURLToPath(import.meta.url)); const { values } = parseArgs({ options: { - input: { type: 'string', default: resolve(__dirname, '../posix-conformance-report.json') }, - exclusions: { type: 'string', default: resolve(__dirname, '../packages/wasmvm/test/posix-exclusions.json') }, - output: { type: 'string', default: resolve(__dirname, '../docs/posix-conformance-report.mdx') }, + input: { type: 'string', default: resolve(__dirname, '../os-test-conformance-report.json') }, + exclusions: { type: 'string', default: resolve(__dirname, '../packages/wasmvm/test/os-test-exclusions.json') }, + output: { type: 'string', default: resolve(__dirname, '../docs/os-test-conformance-report.mdx') }, }, }); @@ -88,12 +88,12 @@ function line(s = '') { // Frontmatter line('---'); -line('title: POSIX Conformance Report'); +line('title: os-test Conformance Report'); line('description: os-test POSIX.1-2024 conformance results for WasmVM.'); line('icon: "chart-bar"'); line('---'); line(); -line('{/* AUTO-GENERATED — do not edit. Run scripts/generate-posix-report.ts */}'); +line('{/* AUTO-GENERATED — do not edit. Run scripts/generate-os-test-report.ts */}'); line(); // Summary table @@ -179,7 +179,7 @@ for (const cat of CATEGORY_ORDER) { const mdx = lines.join('\n'); writeFileSync(outputPath, mdx, 'utf-8'); -console.log(`POSIX Conformance Report generated`); +console.log(`os-test Conformance Report generated`); console.log(` Input: ${inputPath}`); console.log(` Exclusions: ${exclusionsPath}`); console.log(` Output: ${outputPath}`); diff --git a/scripts/import-libc-test.ts b/scripts/import-libc-test.ts new file mode 100644 index 00000000..90d39b35 --- /dev/null +++ b/scripts/import-libc-test.ts @@ -0,0 +1,223 @@ +#!/usr/bin/env -S npx tsx +/** + * Pulls musl libc-test from the Bytecode Alliance mirror and replaces the local source. + * + * Usage: pnpm tsx scripts/import-libc-test.ts --version master + * Downloads the specified branch/tag from GitHub and replaces + * native/wasmvm/c/libc-test/ with the new source. Prints a diff summary + * of added/removed/changed files. + */ + +import { execSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + renameSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +// ── CLI args ──────────────────────────────────────────────────────────── + +const { values } = parseArgs({ + options: { + version: { type: 'string', short: 'v' }, + }, +}); + +if (!values.version) { + console.error('Usage: pnpm tsx scripts/import-libc-test.ts --version '); + console.error(' e.g. --version master or --version v1.0.0'); + process.exit(1); +} + +const version = values.version; + +// Validate version format +const VERSION_RE = /^[a-zA-Z0-9][a-zA-Z0-9._\-/]*$/; +if (!VERSION_RE.test(version)) { + console.error(`Invalid version format: "${version}"`); + process.exit(1); +} + +// ── Paths ─────────────────────────────────────────────────────────────── + +const C_DIR = resolve(__dirname, '../native/wasmvm/c'); +const LIBC_TEST_DIR = join(C_DIR, 'libc-test'); +const CACHE_DIR = join(C_DIR, '.cache/libs'); +const ARCHIVE_PATH = join(CACHE_DIR, 'libc-test.tar.gz'); +const TEMP_DIR = join(C_DIR, 'libc-test-incoming'); +const URL = `https://github.com/bytecodealliance/libc-test/archive/refs/heads/${version}.tar.gz`; +const EXCLUSIONS_PATH = resolve(__dirname, '../packages/wasmvm/test/libc-test-exclusions.json'); + +// ── Helpers ───────────────────────────────────────────────────────────── + +function collectFiles(dir: string, prefix = ''): Set { + const results = new Set(); + if (!existsSync(dir)) return results; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isDirectory()) { + for (const f of collectFiles(join(dir, entry.name), rel)) { + results.add(f); + } + } else { + results.add(rel); + } + } + return results; +} + +function resolveCommitHash(ref: string): string { + try { + const output = execSync( + `git ls-remote https://github.com/bytecodealliance/libc-test.git "${ref}"`, + { stdio: 'pipe', encoding: 'utf-8' }, + ).trim(); + const match = output.match(/^([0-9a-f]{40})/m); + if (match) return match[1]; + } catch { + // Fall through + } + if (/^[0-9a-f]{40}$/.test(ref)) return ref; + console.warn(` Warning: could not resolve commit hash for "${ref}" — using as-is`); + return ref; +} + +// ── Snapshot existing files ───────────────────────────────────────────── + +console.log(`Importing musl libc-test ${version}`); +console.log(` URL: ${URL}`); +console.log(''); + +const oldFiles = collectFiles(LIBC_TEST_DIR); + +// ── Download ──────────────────────────────────────────────────────────── + +console.log('Downloading...'); +mkdirSync(CACHE_DIR, { recursive: true }); + +try { + execSync(`curl -fSL "${URL}" -o "${ARCHIVE_PATH}"`, { stdio: 'pipe' }); +} catch { + console.error(`Download failed.`); + console.error(` URL: ${URL}`); + process.exit(1); +} + +const archiveSize = statSync(ARCHIVE_PATH).size; +console.log(` Downloaded ${(archiveSize / 1024 / 1024).toFixed(1)} MB`); + +// ── Extract to temp dir and validate ──────────────────────────────────── + +console.log('Extracting to temp directory...'); + +if (existsSync(TEMP_DIR)) { + rmSync(TEMP_DIR, { recursive: true, force: true }); +} +mkdirSync(TEMP_DIR, { recursive: true }); + +try { + execSync(`tar -xzf "${ARCHIVE_PATH}" --strip-components=1 -C "${TEMP_DIR}"`, { + stdio: 'pipe', + }); +} catch (err) { + rmSync(TEMP_DIR, { recursive: true, force: true }); + console.error('Extraction failed — existing libc-test/ is untouched.'); + process.exit(1); +} + +// Validate: libc-test has src/ directory with .c files +const tempFiles = collectFiles(TEMP_DIR); +const tempCFiles = [...tempFiles].filter((f) => f.endsWith('.c')); +if (tempCFiles.length === 0) { + rmSync(TEMP_DIR, { recursive: true, force: true }); + console.error('Validation failed: extracted archive contains no .c files.'); + console.error(' Existing libc-test/ is untouched.'); + process.exit(1); +} + +console.log(` Validated: ${tempCFiles.length} .c files found`); + +// ── Swap: remove old, move new into place ─────────────────────────────── + +console.log('Replacing libc-test/ ...'); +if (existsSync(LIBC_TEST_DIR)) { + rmSync(LIBC_TEST_DIR, { recursive: true, force: true }); +} +renameSync(TEMP_DIR, LIBC_TEST_DIR); + +// ── Resolve commit hash and update exclusions metadata ────────────────── + +console.log('Resolving commit hash...'); +const commitHash = resolveCommitHash(version); +console.log(` Commit: ${commitHash}`); + +if (existsSync(EXCLUSIONS_PATH)) { + const exclusions = JSON.parse(readFileSync(EXCLUSIONS_PATH, 'utf-8')); + exclusions.libcTestVersion = version; + exclusions.sourceCommit = commitHash; + exclusions.lastUpdated = new Date().toISOString().slice(0, 10); + writeFileSync(EXCLUSIONS_PATH, JSON.stringify(exclusions, null, 2) + '\n'); + console.log(' Updated libc-test-exclusions.json metadata'); +} + +// ── Diff summary ──────────────────────────────────────────────────────── + +const newFiles = collectFiles(LIBC_TEST_DIR); + +const added: string[] = []; +const removed: string[] = []; + +for (const f of newFiles) { + if (!oldFiles.has(f)) added.push(f); +} +for (const f of oldFiles) { + if (!newFiles.has(f)) removed.push(f); +} + +const cFiles = [...newFiles].filter((f) => f.endsWith('.c')); + +console.log(''); +console.log('Diff Summary'); +console.log('\u2500'.repeat(50)); +console.log(` Previously: ${oldFiles.size} files`); +console.log(` Now: ${newFiles.size} files`); +console.log(` Added: ${added.length} files`); +console.log(` Removed: ${removed.length} files`); +console.log(` C tests: ${cFiles.length} .c files`); + +if (added.length > 0 && added.length <= 50) { + console.log(''); + console.log(' Added files:'); + for (const f of added.sort()) console.log(` + ${f}`); +} + +if (removed.length > 0 && removed.length <= 50) { + console.log(''); + console.log(' Removed files:'); + for (const f of removed.sort()) console.log(` - ${f}`); +} + +if (added.length > 50 || removed.length > 50) { + console.log(''); + console.log(` (${added.length} added / ${removed.length} removed — too many to list)`); +} + +// ── Next steps ────────────────────────────────────────────────────────── + +console.log(''); +console.log('Next steps:'); +console.log(' 1. Rebuild: make -C native/wasmvm/c libc-test libc-test-native'); +console.log(' 2. Test: pnpm vitest run packages/wasmvm/test/libc-test-conformance.test.ts'); +console.log(' 3. Update exclusions: review new failures and update libc-test-exclusions.json'); +console.log(' 4. Validate: pnpm tsx scripts/validate-libc-test-exclusions.ts'); +console.log(' 5. Report: pnpm tsx scripts/generate-libc-test-report.ts'); diff --git a/scripts/import-os-test.ts b/scripts/import-os-test.ts index bee53dda..123afc69 100644 --- a/scripts/import-os-test.ts +++ b/scripts/import-os-test.ts @@ -57,7 +57,7 @@ const CACHE_DIR = join(C_DIR, '.cache/libs'); const ARCHIVE_PATH = join(CACHE_DIR, 'os-test.tar.gz'); const TEMP_DIR = join(C_DIR, 'os-test-incoming'); const URL = `https://gitlab.com/sortix/os-test/-/archive/${version}/os-test-${version}.tar.gz`; -const EXCLUSIONS_PATH = resolve(__dirname, '../packages/wasmvm/test/posix-exclusions.json'); +const EXCLUSIONS_PATH = resolve(__dirname, '../packages/wasmvm/test/os-test-exclusions.json'); // ── Helpers ───────────────────────────────────────────────────────────── @@ -171,7 +171,7 @@ if (existsSync(EXCLUSIONS_PATH)) { exclusions.sourceCommit = commitHash; exclusions.lastUpdated = new Date().toISOString().slice(0, 10); writeFileSync(EXCLUSIONS_PATH, JSON.stringify(exclusions, null, 2) + '\n'); - console.log(' Updated posix-exclusions.json metadata'); + console.log(' Updated os-test-exclusions.json metadata'); } // ── Diff summary ──────────────────────────────────────────────────────── @@ -222,7 +222,7 @@ if (added.length > 50 || removed.length > 50) { console.log(''); console.log('Next steps:'); console.log(' 1. Rebuild: make -C native/wasmvm/c os-test os-test-native'); -console.log(' 2. Test: pnpm vitest run packages/wasmvm/test/posix-conformance.test.ts'); -console.log(' 3. Update exclusions: review new failures and update posix-exclusions.json'); -console.log(' 4. Validate: pnpm tsx scripts/validate-posix-exclusions.ts'); -console.log(' 5. Report: pnpm tsx scripts/generate-posix-report.ts'); +console.log(' 2. Test: pnpm vitest run packages/wasmvm/test/os-test-conformance.test.ts'); +console.log(' 3. Update exclusions: review new failures and update os-test-exclusions.json'); +console.log(' 4. Validate: pnpm tsx scripts/validate-os-test-exclusions.ts'); +console.log(' 5. Report: pnpm tsx scripts/generate-os-test-report.ts'); diff --git a/scripts/validate-libc-test-exclusions.ts b/scripts/validate-libc-test-exclusions.ts new file mode 100644 index 00000000..e9e02e51 --- /dev/null +++ b/scripts/validate-libc-test-exclusions.ts @@ -0,0 +1,118 @@ +#!/usr/bin/env -S npx tsx +/** + * Validates libc-test-exclusions.json for integrity. + * + * Checks: + * 1. Every exclusion key matches a compiled test binary + * 2. Every entry has a non-empty reason string + * 3. Every expected-fail entry has a non-empty issue URL + * 4. Every entry has a valid category from the fixed set + * 5. Every entry has a valid expected value (fail or skip) + * + * Usage: pnpm tsx scripts/validate-libc-test-exclusions.ts + */ + +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { resolve, dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { VALID_EXPECTED, VALID_CATEGORIES } from './conformance-exclusion-schema.js'; +import type { ExclusionEntry } from './conformance-exclusion-schema.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +// ── Paths ────────────────────────────────────────────────────────────── + +const EXCLUSIONS_PATH = resolve(__dirname, '../packages/wasmvm/test/libc-test-exclusions.json'); +const LIBC_TEST_WASM_DIR = resolve(__dirname, '../native/wasmvm/c/build/libc-test'); + +// ── Test discovery ───────────────────────────────────────────────────── + +function discoverTests(dir: string, prefix = ''): string[] { + if (!existsSync(dir)) return []; + const results: string[] = []; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isDirectory()) { + results.push(...discoverTests(join(dir, entry.name), rel)); + } else { + results.push(rel); + } + } + return results.sort(); +} + +// ── Load data ────────────────────────────────────────────────────────── + +const exclusionsData = JSON.parse(readFileSync(EXCLUSIONS_PATH, 'utf-8')); +const exclusions: Record = exclusionsData.exclusions; +const allTests = discoverTests(LIBC_TEST_WASM_DIR); + +// ── Validation ───────────────────────────────────────────────────────── + +const errors: string[] = []; +const warnings: string[] = []; + +for (const [key, entry] of Object.entries(exclusions)) { + // Valid expected value + if (!VALID_EXPECTED.includes(entry.expected as any)) { + errors.push(`[${key}] Invalid expected "${entry.expected}" — must be one of: ${VALID_EXPECTED.join(', ')}`); + } + + // Valid category + if (!VALID_CATEGORIES.includes(entry.category as any)) { + errors.push(`[${key}] Invalid category "${entry.category}" — must be one of: ${VALID_CATEGORIES.join(', ')}`); + } + + // Non-empty reason + if (!entry.reason || entry.reason.trim().length === 0) { + errors.push(`[${key}] Missing or empty reason`); + } + + // expected-fail entries must have issue URL + if (entry.expected === 'fail') { + if (!entry.issue || entry.issue.trim().length === 0) { + errors.push(`[${key}] Expected "fail" but missing issue URL`); + } else if (!/^https:\/\/github\.com\/rivet-dev\/secure-exec\/issues\/\d+$/.test(entry.issue)) { + errors.push(`[${key}] Issue URL must match https://github.com/rivet-dev/secure-exec/issues/, got: ${entry.issue}`); + } + } + + // Key must match a compiled test binary + if (allTests.length > 0 && !allTests.includes(key)) { + warnings.push(`[${key}] Does not match any compiled test binary`); + } +} + +// ── Output ───────────────────────────────────────────────────────────── + +if (allTests.length === 0) { + console.log('Warning: No compiled libc-test WASM binaries found — skipping binary checks'); + console.log(` Build them with: make -C native/wasmvm/c libc-test`); +} + +const entryCount = Object.keys(exclusions).length; +const skipCount = Object.values(exclusions).filter((e) => e.expected === 'skip').length; +const failCount = Object.values(exclusions).filter((e) => e.expected === 'fail').length; + +console.log(`\nlibc-test Exclusion List Validation`); +console.log('─'.repeat(50)); +console.log(`Entries: ${entryCount}`); +console.log(`Expected fail: ${failCount}`); +console.log(`Skip (unrunnable): ${skipCount}`); +console.log(`Test binaries: ${allTests.length}`); +console.log(''); + +if (warnings.length > 0) { + console.log(`Warnings (${warnings.length}):`); + for (const w of warnings) console.log(` ! ${w}`); + console.log(''); +} + +if (errors.length > 0) { + console.log(`Errors (${errors.length}):`); + for (const e of errors) console.log(` x ${e}`); + console.log(''); + process.exit(1); +} + +console.log('All checks passed'); diff --git a/scripts/validate-posix-exclusions.ts b/scripts/validate-os-test-exclusions.ts similarity index 92% rename from scripts/validate-posix-exclusions.ts rename to scripts/validate-os-test-exclusions.ts index 3d9d5779..3455e0ca 100644 --- a/scripts/validate-posix-exclusions.ts +++ b/scripts/validate-os-test-exclusions.ts @@ -1,6 +1,6 @@ #!/usr/bin/env -S npx tsx /** - * Validates posix-exclusions.json for integrity. + * Validates os-test-exclusions.json for integrity. * * Checks: * 1. Every exclusion key matches a compiled test binary @@ -9,20 +9,20 @@ * 4. Every entry has a valid category from the fixed set * 5. Every entry has a valid expected value (fail or skip) * - * Usage: pnpm tsx scripts/validate-posix-exclusions.ts + * Usage: pnpm tsx scripts/validate-os-test-exclusions.ts */ import { existsSync, readdirSync, readFileSync } from 'node:fs'; import { resolve, dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { VALID_EXPECTED, VALID_CATEGORIES } from './posix-exclusion-schema.js'; -import type { ExclusionEntry } from './posix-exclusion-schema.js'; +import { VALID_EXPECTED, VALID_CATEGORIES } from './conformance-exclusion-schema.js'; +import type { ExclusionEntry } from './conformance-exclusion-schema.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); // ── Paths ────────────────────────────────────────────────────────────── -const EXCLUSIONS_PATH = resolve(__dirname, '../packages/wasmvm/test/posix-exclusions.json'); +const EXCLUSIONS_PATH = resolve(__dirname, '../packages/wasmvm/test/os-test-exclusions.json'); const OS_TEST_WASM_DIR = resolve(__dirname, '../native/wasmvm/c/build/os-test'); // ── Test discovery ───────────────────────────────────────────────────── @@ -94,7 +94,7 @@ const entryCount = Object.keys(exclusions).length; const skipCount = Object.values(exclusions).filter((e) => e.expected === 'skip').length; const failCount = Object.values(exclusions).filter((e) => e.expected === 'fail').length; -console.log(`\nPOSIX Exclusion List Validation`); +console.log(`\nos-test Exclusion List Validation`); console.log('─'.repeat(50)); console.log(`Entries: ${entryCount}`); console.log(`Expected fail: ${failCount}`);