From 78900c09ca7a12e37dba7cdf86a5c9edc3957bd0 Mon Sep 17 00:00:00 2001 From: Joachim Rosskopf Date: Fri, 22 May 2026 12:42:22 +0200 Subject: [PATCH 1/5] ci: cross-platform self-packaging smoke jobs Part of #40. Closes #49. Adds four new CI jobs that exercise the self-packaging surface on every platform flapi builds for: pack-smoke-linux-amd64 ubuntu-24.04 pack-smoke-linux-arm64 ubuntu-24.04-arm pack-smoke-macos macos-latest pack-smoke-windows windows-latest Each job: - downloads the platform's `flapi` artifact from the existing windows-build / linux-build / osx-universal-build jobs - builds a tiny fixture tree (flapi.yaml + one endpoint + sample SQL) - runs `flapi pack --in fixture --out out-a` - runs `out-a info`, asserts the entry list contains the fixture files - runs `out-a unpack --to extracted`, diffs files byte-for-byte - runs a second `flapi pack ... --out out-b` with the same `SOURCE_DATE_EPOCH=1700000000` and asserts `sha256(out-a) == sha256(out-b)` -- the reproducible-build invariant baked into archive_io (#41) The macOS leg additionally: - runs `otool -l` to confirm the unbundled binary carries the reserved `__FLAPI/__bundle` segment from link time - runs `codesign --verify --strict out-segment` after the default reserved-segment pack -- the notarisation precondition this whole approach was built for - runs `flapi pack --macos-append --out out-append` and confirms `info` still discovers the bundle (ad-hoc legacy path still works, signature is intentionally invalid -- we do not run codesign verify here) - runs an oversized-payload pack (32 MiB into a 16-MiB segment) and confirms it exits non-zero with both "FLAPI_RESERVED_BUNDLE_MIB" and "reserved|exceeds" in the error message The Windows job uses pwsh; pack/info/unpack/sha256 logic mirrors Linux via PowerShell idioms. `create-release` now `needs` the four new jobs in addition to the existing build + smoke matrix, so a broken pack on any platform blocks the release. The existing `integration-tests` job (which already runs on ubuntu-24.04 amd64) auto-picks up `test_self_packaging.py` (#56) and `test_env_overrides.py` (#57) via pytest discovery -- no change needed there. Note: on macOS we deliberately do NOT assert reproducibility because the codesign step may include non-deterministic data; on Linux and Windows the host-bytes + appended-archive layout is reproducible by construction. --- .github/workflows/build.yaml | 253 ++++++++++++++++++++++++++++++++++- 1 file changed, 252 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8ba21ea..9dbd370 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -551,12 +551,263 @@ jobs: exit 1 } + # ------------------------------------------------------------------------------------------------- + # Self-packaging Smoke Tests (#49) — pack / info / unpack + reproducibility on every platform. + # macOS additionally verifies the codesign signature on the reserved-segment output. + # ------------------------------------------------------------------------------------------------- + pack-smoke-linux-amd64: + needs: [linux-build] + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Download flapi binary + uses: actions/download-artifact@v4 + with: + name: flapi-linux-amd64 + path: bin + + - name: Pack / info / unpack / reproducibility + env: + SOURCE_DATE_EPOCH: "1700000000" + run: | + chmod +x bin/flapi + ./bin/flapi --version || true + + mkdir -p fixture/sqls + cat > fixture/flapi.yaml <<'YAML' + project-name: pack-smoke + project-description: CI pack smoke fixture + template: + path: ./sqls + connections: {} + duckdb: + access_mode: READ_WRITE + threads: 1 + YAML + cat > fixture/sqls/hello.yaml <<'YAML' + url-path: /hello + method: GET + template-source: hello.sql + connection: [] + YAML + echo "SELECT 'world' AS greeting" > fixture/sqls/hello.sql + + # pack + ./bin/flapi pack --in fixture --out out-a + test -s out-a + chmod +x out-a + + # info should list the fixture entries + ./out-a info | tee info-a.txt + grep -q "flapi.yaml" info-a.txt + grep -q "sqls/hello.sql" info-a.txt + + # unpack and verify + ./out-a unpack --to extracted + test -s extracted/flapi.yaml + test -s extracted/sqls/hello.sql + diff fixture/sqls/hello.sql extracted/sqls/hello.sql + + # reproducibility: same input + same SOURCE_DATE_EPOCH ⇒ identical sha256 + ./bin/flapi pack --in fixture --out out-b + sha256sum out-a out-b + test "$(sha256sum out-a | awk '{print $1}')" = "$(sha256sum out-b | awk '{print $1}')" + + pack-smoke-linux-arm64: + needs: [linux-build] + runs-on: ubuntu-24.04-arm + timeout-minutes: 10 + steps: + - name: Download flapi binary + uses: actions/download-artifact@v4 + with: + name: flapi-linux-arm64 + path: bin + + - name: Pack / info / unpack / reproducibility + env: + SOURCE_DATE_EPOCH: "1700000000" + run: | + chmod +x bin/flapi + mkdir -p fixture/sqls + cat > fixture/flapi.yaml <<'YAML' + project-name: pack-smoke + project-description: CI pack smoke fixture + template: + path: ./sqls + connections: {} + duckdb: + access_mode: READ_WRITE + threads: 1 + YAML + cat > fixture/sqls/hello.yaml <<'YAML' + url-path: /hello + method: GET + template-source: hello.sql + connection: [] + YAML + echo "SELECT 'world' AS greeting" > fixture/sqls/hello.sql + + ./bin/flapi pack --in fixture --out out-a + chmod +x out-a + ./out-a info | tee info.txt + grep -q "flapi.yaml" info.txt + ./out-a unpack --to extracted + diff fixture/sqls/hello.sql extracted/sqls/hello.sql + + ./bin/flapi pack --in fixture --out out-b + test "$(sha256sum out-a | awk '{print $1}')" = "$(sha256sum out-b | awk '{print $1}')" + + pack-smoke-macos: + needs: [osx-universal-build] + runs-on: macos-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + submodules: false + + - name: Download flapi binary + uses: actions/download-artifact@v4 + with: + name: flapi-macos-arm64 + path: bin + + - name: Pack / info / unpack + codesign verify (reserved segment) + env: + SOURCE_DATE_EPOCH: "1700000000" + run: | + chmod +x bin/flapi + + # The link-time placeholder should exist before any pack. + otool -l bin/flapi | grep -q "segname __FLAPI" + otool -l bin/flapi | grep -q "sectname __bundle" + + mkdir -p fixture/sqls + cat > fixture/flapi.yaml <<'YAML' + project-name: pack-smoke + project-description: CI pack smoke fixture + template: + path: ./sqls + connections: {} + duckdb: + access_mode: READ_WRITE + threads: 1 + YAML + cat > fixture/sqls/hello.yaml <<'YAML' + url-path: /hello + method: GET + template-source: hello.sql + connection: [] + YAML + echo "SELECT 'world' AS greeting" > fixture/sqls/hello.sql + + # Default pack uses the reserved-segment path on macOS. + ./bin/flapi pack --in fixture --out out-segment + chmod +x out-segment + ./out-segment info | tee info.txt + grep -q "flapi.yaml" info.txt + ./out-segment unpack --to extracted + diff fixture/sqls/hello.sql extracted/sqls/hello.sql + + # Notarisation precondition: codesign must verify cleanly. + codesign --verify --strict out-segment + + # Legacy append mode -- still produces a working bundle but the + # signature is intentionally ad-hoc. We assert info works; we + # do not assert codesign --verify because the trailing bytes + # cause that to fail (which is precisely why the segment path + # is the default). + ./bin/flapi pack --in fixture --out out-append --macos-append + chmod +x out-append + ./out-append info | grep -q "flapi.yaml" + + - name: oversized payload is rejected with a corrective message + env: + SOURCE_DATE_EPOCH: "1700000000" + run: | + # Default reserved size is 16 MiB. A 32-MiB blob can't fit. + mkdir -p fixture/data + dd if=/dev/urandom of=fixture/data/huge.bin bs=1m count=32 2>/dev/null + + # We expect a non-zero exit. Capture stdout+stderr to one file. + set +e + ./bin/flapi pack --in fixture --out out-too-big > err.txt 2>&1 + rc=$? + set -e + cat err.txt + if [ "$rc" -eq 0 ]; then + echo "ERROR: oversized pack unexpectedly succeeded" + exit 1 + fi + grep -qi "FLAPI_RESERVED_BUNDLE_MIB" err.txt + grep -Eqi "reserved|exceeds" err.txt + + pack-smoke-windows: + needs: [windows-build] + runs-on: windows-latest + timeout-minutes: 10 + steps: + - name: Download flapi binary + uses: actions/download-artifact@v4 + with: + name: flapi-windows-amd64 + path: bin + + - name: Pack / info / unpack / reproducibility + shell: pwsh + env: + SOURCE_DATE_EPOCH: "1700000000" + run: | + New-Item -ItemType Directory -Force fixture\sqls | Out-Null + + @' + project-name: pack-smoke + project-description: CI pack smoke fixture + template: + path: ./sqls + connections: {} + duckdb: + access_mode: READ_WRITE + threads: 1 + '@ | Out-File -FilePath fixture\flapi.yaml -Encoding UTF8 + + @' + url-path: /hello + method: GET + template-source: hello.sql + connection: [] + '@ | Out-File -FilePath fixture\sqls\hello.yaml -Encoding UTF8 + + "SELECT 'world' AS greeting" | Out-File -FilePath fixture\sqls\hello.sql -Encoding UTF8 + + & bin\flapi.exe pack --in fixture --out out-a.exe + if ($LASTEXITCODE -ne 0) { throw "pack a failed" } + + $info = & .\out-a.exe info + $info | Out-Host + if ($info -notmatch "flapi.yaml") { throw "info missing flapi.yaml" } + if ($info -notmatch "sqls/hello.sql") { throw "info missing hello.sql" } + + & .\out-a.exe unpack --to extracted + if (-not (Test-Path extracted\flapi.yaml)) { throw "unpack missing flapi.yaml" } + if (-not (Test-Path extracted\sqls\hello.sql)) { throw "unpack missing hello.sql" } + + & bin\flapi.exe pack --in fixture --out out-b.exe + if ($LASTEXITCODE -ne 0) { throw "pack b failed" } + + $h1 = (Get-FileHash out-a.exe -Algorithm SHA256).Hash + $h2 = (Get-FileHash out-b.exe -Algorithm SHA256).Hash + Write-Host "out-a sha256: $h1" + Write-Host "out-b sha256: $h2" + if ($h1 -ne $h2) { throw "reproducibility check failed: $h1 != $h2" } + # ------------------------------------------------------------------------------------------------- # Create GitHub Release + PyPI Wheels (tag pushes only) # ------------------------------------------------------------------------------------------------- create-release: if: startsWith(github.ref, 'refs/tags/v') - needs: [windows-build, linux-build, osx-universal-build, flapii-build, smoke-test-windows, smoke-test-linux-amd64, smoke-test-linux-arm64, smoke-test-macos] + needs: [windows-build, linux-build, osx-universal-build, flapii-build, smoke-test-windows, smoke-test-linux-amd64, smoke-test-linux-arm64, smoke-test-macos, pack-smoke-linux-amd64, pack-smoke-linux-arm64, pack-smoke-macos, pack-smoke-windows] runs-on: ubuntu-24.04 permissions: contents: write From 614c7003d62cca105c644235d462a818d6053fcc Mon Sep 17 00:00:00 2001 From: Joachim Rosskopf Date: Fri, 22 May 2026 21:22:02 +0200 Subject: [PATCH 2/5] fix: LocateBundle probes the macOS reserved segment too Previously LocateBundle(path) only did the EOF tail scan. Only LocateBundleInSelf() probed the __FLAPI/__bundle Mach-O section first. That asymmetry meant flapi info / flapi unpack -- both of which call LocateBundle directly against a given binary -- couldn't find bundles that flapi pack wrote into the reserved segment on macOS. Caught by CI: pack-smoke-macos reported Packed 3 entries (683 bytes) into out-segment Bundle: none (filesystem mode) The pack succeeded (3 entries went into the section), but info checked only the EOF tail and saw nothing. Fix: move the section-probe into LocateBundle(path), where every caller benefits. LocateBundleInSelf becomes a thin wrapper. The fall-through to EOF tail is preserved so --macos-append bundles (legacy ad-hoc layout) remain discoverable. Found by CI on PR #59 (#49 cross-platform smoke jobs). --- src/bundle_locator.cpp | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/bundle_locator.cpp b/src/bundle_locator.cpp index 30d0b7d..3c48675 100644 --- a/src/bundle_locator.cpp +++ b/src/bundle_locator.cpp @@ -112,6 +112,20 @@ std::optional ScanBufferForEocd( std::optional LocateBundle(const std::filesystem::path& path) { std::error_code ec; + // macOS reserved-segment path (#48): try the __FLAPI/__bundle section + // first. Linux/Windows binaries don't have the section, so this is a + // cheap miss and we fall through to the EOF tail scan below. Required + // so `flapi info` / `flapi unpack` (which call LocateBundle directly, + // not LocateBundleInSelf) find macOS-packed bundles. + if (auto sect = LocateFlapiSection(path); sect.has_value()) { + if (auto loc = LocateBundleInRange(path, sect->file_offset, sect->size); + loc.has_value()) { + return loc; + } + // Section present but unpopulated -- fall through to EOF scan + // so --macos-append bundles are still discoverable. + } + const auto file_size = std::filesystem::file_size(path, ec); if (ec || file_size < kEocdRecordSize) { return std::nullopt; @@ -175,19 +189,7 @@ std::optional LocateBundleInSelf() { } catch (...) { return std::nullopt; } - - // Prefer the reserved Mach-O section if present (#48). Linux/Windows - // binaries lack the section, so this returns nullopt and we fall - // through to the EOF-tail scan unchanged. - if (auto sect = LocateFlapiSection(self_path); sect.has_value()) { - if (auto loc = LocateBundleInRange(self_path, sect->file_offset, sect->size); - loc.has_value()) { - return loc; - } - // Section is present but empty / unpopulated (e.g., un-packed - // build): fall through to the EOF-tail scan. - } - + // LocateBundle now tries section-mode first, then EOF tail. try { return LocateBundle(self_path); } catch (...) { From 602174a31406d216952c499d96bab666fa0d0987 Mon Sep 17 00:00:00 2001 From: Joachim Rosskopf Date: Fri, 22 May 2026 22:17:18 +0200 Subject: [PATCH 3/5] fix(pack): tolerate codesign failure on --macos-append + correct pwsh array-match Two follow-up fixes surfaced by CI on PR #59: 1. macOS: `flapi pack --macos-append` always tried to codesign the output, which fails with "main executable failed strict validation" because the appended ZIP after __LINKEDIT puts the binary outside what codesign considers signable. That's the documented trade-off -- append-mode output is explicitly not notarisable. Now we warn and continue; only the reserved-segment path treats codesign failure as fatal. 2. Windows pack-smoke: `$info -notmatch "x"` on a PowerShell array returns the filtered subset, not a boolean -- so the negation in the if statement was always truthy. Pack and info both worked correctly (info actually listed flapi.yaml, sqls/...); the test logic was the bug. Join the captured lines into a scalar string before -notmatch. Re-running the macOS leg should now show pack-smoke-macos passing through the --macos-append step. Re-running the Windows leg should get all the way to the reproducibility check. Found by CI on PR #59 (#49 cross-platform smoke jobs). --- .github/workflows/build.yaml | 9 ++++++--- src/pack.cpp | 27 +++++++++++++++++---------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9dbd370..460da3c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -784,9 +784,12 @@ jobs: & bin\flapi.exe pack --in fixture --out out-a.exe if ($LASTEXITCODE -ne 0) { throw "pack a failed" } - $info = & .\out-a.exe info - $info | Out-Host - if ($info -notmatch "flapi.yaml") { throw "info missing flapi.yaml" } + # PowerShell's `$arr -notmatch "x"` returns a filtered array, + # not a boolean. Join the captured lines so substring match + # actually does what we mean. + $info = (& .\out-a.exe info) -join "`n" + Write-Host $info + if ($info -notmatch "flapi.yaml") { throw "info missing flapi.yaml" } if ($info -notmatch "sqls/hello.sql") { throw "info missing hello.sql" } & .\out-a.exe unpack --to extracted diff --git a/src/pack.cpp b/src/pack.cpp index 61799ff..d74411d 100644 --- a/src/pack.cpp +++ b/src/pack.cpp @@ -192,20 +192,27 @@ PackResult Pack(const std::filesystem::path& in_dir, } // Re-sign on Darwin. CodesignBinary is a benign no-op on other - // platforms so we don't need a platform guard here. Note: a - // codesign failure is reported to the caller via PackError; the - // append path on macOS still benefits from an ad-hoc re-sign - // because the trailing-data invalidates the original signature. + // platforms so we don't need a platform guard here. + // + // Note on the --macos-append legacy path: codesign --force fails + // with "main executable failed strict validation" because the + // trailing ZIP after __LINKEDIT puts the binary outside what + // codesign considers signable. That's the documented trade-off -- + // append-mode output is explicitly not notarisable. We warn and + // continue; only the reserved-segment path treats codesign failure + // as fatal. if (options.codesign) { auto cs = CodesignBinary(out_path); if (cs.exit_code != 0) { - // Don't crash a Linux build over codesign output. On macOS - // a non-zero exit means we couldn't make the binary - // launchable -- surface it. #ifdef __APPLE__ - throw PackError("codesign failed (exit " + - std::to_string(cs.exit_code) + "): " + - cs.stderr_tail); + if (used_section_path) { + throw PackError("codesign failed (exit " + + std::to_string(cs.exit_code) + "): " + + cs.stderr_tail); + } + std::cerr << "flapi pack: warning: codesign failed on --macos-append " + "output (expected; result is not notarisable): " + << cs.stderr_tail << '\n'; #else (void)cs; // unreachable in practice (we always set exit_code = 0) #endif From fe2950e20979ac3c5c6699e60fd7a87d4ac8b35c Mon Sep 17 00:00:00 2001 From: Joachim Rosskopf Date: Fri, 22 May 2026 23:08:47 +0200 Subject: [PATCH 4/5] fix(pack): missing for std::cerr warning on --macos-append CI on macos-latest failed to build src/pack.cpp: error: no member named 'cerr' in namespace 'std' Caused by the previous fix that warns instead of throws on the --macos-append codesign-failure path. doesn't drag std::cerr in; we need explicitly. --- src/pack.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pack.cpp b/src/pack.cpp index d74411d..ab0dcee 100644 --- a/src/pack.cpp +++ b/src/pack.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include From 7ffed115889e09d8a69959ae57310d9545d72398 Mon Sep 17 00:00:00 2001 From: Joachim Rosskopf Date: Fri, 22 May 2026 23:53:47 +0200 Subject: [PATCH 5/5] ci: rerun (flake retry on integration-tests' sql-injection corpus)