diff --git a/.github/sbom-policy.json b/.github/sbom-policy.json new file mode 100644 index 0000000..f0e3326 --- /dev/null +++ b/.github/sbom-policy.json @@ -0,0 +1,7 @@ +{ + "max_added": 50, + "max_removed": 50, + "deny_licenses": [], + "require_licenses": true, + "deny_duplicates": true +} diff --git a/.github/workflows/sbom-generate-upload.yml b/.github/workflows/sbom-generate.yml similarity index 68% rename from .github/workflows/sbom-generate-upload.yml rename to .github/workflows/sbom-generate.yml index ee3a2bf..b8bfd3d 100644 --- a/.github/workflows/sbom-generate-upload.yml +++ b/.github/workflows/sbom-generate.yml @@ -1,26 +1,26 @@ -name: Generate and upload SBOMs +name: Generate SBOMs -# Runs after OCI images are built, or on direct pushes that touch image sources. -# -# Prerequisites (one-time setup): -# 1. Add SBOMIFY_TOKEN secret to the repository (Settings → Secrets → Actions). -# Generate the token at https://app.sbomify.com. -# 2. Create one sbomify component per image and replace the placeholder -# sbomify_component_id values in the matrix below with the real IDs. +# Generates and enriches CycloneDX SBOMs for all Nix-built OCI images. +# Enriched SBOMs are uploaded as artifacts for downstream workflows +# (quality gate on PRs, upload to sbomify on merge). on: - workflow_run: - workflows: ["Build and publish OCI images"] - types: [completed] + pull_request: branches: [main] + paths: + - images/** + - deployments/sbomify/** + - flake.nix + - flake.lock + - bin/patch-sbom-root push: - branches: - - main + branches: [main] paths: - images/** - deployments/sbomify/** - flake.nix - flake.lock + - bin/patch-sbom-root workflow_dispatch: permissions: @@ -28,11 +28,7 @@ permissions: contents: read jobs: - generate-and-upload: - # Skip if triggered by a failed workflow_run - if: >- - github.event_name != 'workflow_run' || - github.event.workflow_run.conclusion == 'success' + generate: runs-on: blacksmith-4vcpu-ubuntu-2404 strategy: fail-fast: false @@ -41,34 +37,42 @@ jobs: - name: postgres sbom_package: postgres-sbom nixpkg: postgresql_17 + license: PostgreSQL sbomify_component_id: M8rixM6mMEPe - name: redis sbom_package: redis-sbom nixpkg: redis + license: AGPL-3.0-only sbomify_component_id: ABBCcw2YiYrG - name: minio sbom_package: minio-sbom nixpkg: minio + license: AGPL-3.0-or-later sbomify_component_id: PLACEHOLDER_MINIO - name: minio-client sbom_package: minio-client-sbom nixpkg: minio-client + license: Apache-2.0 sbomify_component_id: PLACEHOLDER_MINIO_CLIENT - name: sbomify-app sbom_package: sbomify-app-sbom nixpkg: python313 + license: Apache-2.0 sbomify_component_id: VP42I4XQgpDE - name: sbomify-keycloak sbom_package: sbomify-keycloak-sbom nixpkg: keycloak + license: Apache-2.0 sbomify_component_id: N4agQD8pvej8 - name: sbomify-caddy-dev sbom_package: sbomify-caddy-dev-sbom nixpkg: caddy + license: Apache-2.0 sbomify_component_id: zYDq6NtrOBuo - name: sbomify-minio-init sbom_package: sbomify-minio-init-sbom nixpkg: minio-client + license: Apache-2.0 sbomify_component_id: PLACEHOLDER_SBOMIFY_MINIO_INIT steps: @@ -90,7 +94,6 @@ jobs: id: version run: | if [[ "${{ matrix.image.name }}" == sbomify-* ]]; then - # sbomify images use the pinned sbomify source version VERSION=$(grep 'github:sbomify/sbomify/' flake.nix | grep -oP '/v\K[0-9.]+') else VERSION=$(nix eval --raw nixpkgs#${{ matrix.image.nixpkg }}.version) @@ -102,7 +105,19 @@ jobs: nix build .#${{ matrix.image.sbom_package }} cp -L result ${{ matrix.image.name }}.cdx.json - - name: Generate and Upload SBOM to sbomify + - name: Patch root component metadata + run: | + VERSION="${{ steps.version.outputs.upstream }}" + SBOM="${{ matrix.image.name }}.cdx.json" + + bin/patch-sbom-root \ + --name "wellmaintained/packages/${{ matrix.image.name }}-image" \ + --version "$VERSION" \ + --purl "pkg:docker/wellmaintained/packages/${{ matrix.image.name }}@${VERSION}" \ + --license "${{ matrix.image.license }}" \ + < "$SBOM" > "${SBOM}.tmp" && mv "${SBOM}.tmp" "$SBOM" + + - name: Enrich SBOM if: ${{ !startsWith(matrix.image.sbomify_component_id, 'PLACEHOLDER') }} uses: sbomify/sbomify-action@master continue-on-error: true @@ -115,12 +130,17 @@ jobs: SBOM_FORMAT: cyclonedx AUGMENT: true ENRICH: true - UPLOAD: true + UPLOAD: false OUTPUT_FILE: ${{ matrix.image.name }}.enriched.cdx.json - - name: Upload enriched SBOM as artifact + - name: Fall back to raw SBOM if enrichment skipped + run: | + if [ ! -f "${{ matrix.image.name }}.enriched.cdx.json" ]; then + cp "${{ matrix.image.name }}.cdx.json" "${{ matrix.image.name }}.enriched.cdx.json" + fi + + - name: Upload enriched SBOM uses: actions/upload-artifact@v6 with: name: sbom-${{ matrix.image.name }} path: ${{ matrix.image.name }}.enriched.cdx.json - if-no-files-found: warn diff --git a/.github/workflows/sbom-quality-gate.yml b/.github/workflows/sbom-quality-gate.yml new file mode 100644 index 0000000..a2e67ae --- /dev/null +++ b/.github/workflows/sbom-quality-gate.yml @@ -0,0 +1,174 @@ +name: SBOM quality gate + +# Scores the PR's SBOMs and compares against the baseline from main. +# Blocks merging if any image's quality score regresses. +# +# Requires the "Generate SBOMs" workflow to have run first on this PR +# (it produces the enriched SBOM artifacts we score here). + +on: + workflow_run: + workflows: ["Generate SBOMs"] + types: [completed] + +permissions: + actions: read + contents: read + pull-requests: write + +jobs: + score: + if: >- + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + runs-on: blacksmith-4vcpu-ubuntu-2404 + strategy: + fail-fast: false + matrix: + image: + - name: postgres + - name: redis + - name: minio + - name: minio-client + - name: sbomify-app + - name: sbomify-keycloak + - name: sbomify-caddy-dev + - name: sbomify-minio-init + + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Install sbomqs + run: | + curl -sSfL https://github.com/interlynk-io/sbomqs/releases/latest/download/sbomqs-linux-amd64 \ + -o /usr/local/bin/sbomqs + chmod +x /usr/local/bin/sbomqs + + - name: Install sbomlyze + run: | + curl -sSfL https://raw.githubusercontent.com/rezmoss/sbomlyze/main/install.sh | sh + mv ./sbomlyze /usr/local/bin/sbomlyze + + - name: Download PR SBOM + uses: actions/download-artifact@v4 + with: + name: sbom-${{ matrix.image.name }} + path: current/ + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + + - name: Score current SBOM + run: | + mkdir -p results + SBOM=$(ls current/*.cdx.json | head -1) + bin/sbom-score --image "${{ matrix.image.name }}" "$SBOM" \ + > results/score-${{ matrix.image.name }}.json + env: + PATH: /usr/local/bin:$PATH + + - name: Fetch baseline SBOM + id: baseline + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: | + # Find the latest successful Generate SBOMs run on main + RUN_ID=$(gh run list \ + --workflow="Generate SBOMs" \ + --branch=main \ + --status=success \ + --limit=1 \ + --json databaseId \ + --jq '.[0].databaseId') + + if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then + echo "has_baseline=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + gh run download "$RUN_ID" \ + --name "sbom-${{ matrix.image.name }}" \ + --dir baseline/ || { + echo "has_baseline=false" >> "$GITHUB_OUTPUT" + exit 0 + } + + echo "has_baseline=true" >> "$GITHUB_OUTPUT" + + - name: Score baseline SBOM + if: steps.baseline.outputs.has_baseline == 'true' + run: | + SBOM=$(ls baseline/*.cdx.json | head -1) + bin/sbom-score --image "${{ matrix.image.name }}" "$SBOM" \ + > results/baseline-${{ matrix.image.name }}.json + + - name: Compare SBOMs + if: steps.baseline.outputs.has_baseline == 'true' + run: | + BASELINE=$(ls baseline/*.cdx.json | head -1) + CURRENT=$(ls current/*.cdx.json | head -1) + bin/sbom-compare \ + --baseline "$BASELINE" \ + --current "$CURRENT" \ + --image "${{ matrix.image.name }}" \ + --policy .github/sbom-policy.json \ + > results/compare-${{ matrix.image.name }}.json + + - name: Upload results + uses: actions/upload-artifact@v6 + with: + name: sbom-qg-${{ matrix.image.name }} + path: results/ + + report: + needs: score + if: always() && needs.score.result != 'skipped' + runs-on: blacksmith-4vcpu-ubuntu-2404 + permissions: + pull-requests: write + actions: read + + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Download all results + uses: actions/download-artifact@v4 + with: + pattern: sbom-qg-* + merge-multiple: true + path: results/ + + - name: Generate report + id: report + run: | + bin/sbom-report --scores-dir results/ > pr-comment.md 2>report-stderr.txt || { + cat report-stderr.txt >&2 + echo "failed=true" >> "$GITHUB_OUTPUT" + } + + - name: Resolve PR number + id: pr + env: + GH_TOKEN: ${{ github.token }} + run: | + # The workflow_run event doesn't directly give us the PR number. + # Look it up from the head branch. + HEAD_BRANCH="${{ github.event.workflow_run.head_branch }}" + PR_NUMBER=$(gh pr list --head "$HEAD_BRANCH" --json number --jq '.[0].number') + echo "number=${PR_NUMBER}" >> "$GITHUB_OUTPUT" + + - name: Post PR comment + if: steps.pr.outputs.number + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: sbom-quality-gate + number: ${{ steps.pr.outputs.number }} + path: pr-comment.md + + - name: Fail on regression + if: steps.report.outputs.failed == 'true' + run: | + echo "SBOM quality regression detected. See PR comment for details." + exit 1 diff --git a/.github/workflows/sbom-upload.yml b/.github/workflows/sbom-upload.yml new file mode 100644 index 0000000..b11746c --- /dev/null +++ b/.github/workflows/sbom-upload.yml @@ -0,0 +1,59 @@ +name: Upload SBOMs to sbomify + +# Uploads enriched SBOMs to sbomify after they are generated on main. +# Only runs when the Generate SBOMs workflow completes successfully on main. + +on: + workflow_run: + workflows: ["Generate SBOMs"] + types: [completed] + branches: [main] + +permissions: + actions: read + contents: read + +jobs: + upload: + if: github.event.workflow_run.conclusion == 'success' + runs-on: blacksmith-4vcpu-ubuntu-2404 + strategy: + fail-fast: false + matrix: + image: + - name: postgres + sbomify_component_id: M8rixM6mMEPe + - name: redis + sbomify_component_id: ABBCcw2YiYrG + - name: minio + sbomify_component_id: PLACEHOLDER_MINIO + - name: minio-client + sbomify_component_id: PLACEHOLDER_MINIO_CLIENT + - name: sbomify-app + sbomify_component_id: VP42I4XQgpDE + - name: sbomify-keycloak + sbomify_component_id: N4agQD8pvej8 + - name: sbomify-caddy-dev + sbomify_component_id: zYDq6NtrOBuo + - name: sbomify-minio-init + sbomify_component_id: PLACEHOLDER_SBOMIFY_MINIO_INIT + + steps: + - name: Download enriched SBOM + if: ${{ !startsWith(matrix.image.sbomify_component_id, 'PLACEHOLDER') }} + uses: actions/download-artifact@v4 + with: + name: sbom-${{ matrix.image.name }} + path: sbom/ + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + + - name: Upload to sbomify + if: ${{ !startsWith(matrix.image.sbomify_component_id, 'PLACEHOLDER') }} + uses: sbomify/sbomify-action@master + env: + TOKEN: ${{ secrets.SBOMIFY_TOKEN }} + COMPONENT_ID: ${{ matrix.image.sbomify_component_id }} + SBOM_FILE: sbom/${{ matrix.image.name }}.enriched.cdx.json + SBOM_FORMAT: cyclonedx + UPLOAD: true diff --git a/.shellspec b/.shellspec new file mode 100644 index 0000000..d567ecf --- /dev/null +++ b/.shellspec @@ -0,0 +1,12 @@ +--require spec_helper + +## Default kcov (coverage) options +# --kcov-options "--include-path=. --path-strip-level=1" +# --kcov-options "--include-pattern=.sh" +# --kcov-options "--exclude-pattern=/.shellspec,/spec/,/coverage/,/report/" + +## Example: Include script "myprog" with no extension +# --kcov-options "--include-pattern=.sh,myprog" + +## Example: Only specified files/directories +# --kcov-options "--include-pattern=myprog,/lib/" diff --git a/bin/generate-sboms.sh b/bin/generate-sboms.sh index ad43bb0..3eb968a 100755 --- a/bin/generate-sboms.sh +++ b/bin/generate-sboms.sh @@ -8,15 +8,16 @@ REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" SBOM_DIR="${REPO_ROOT}/sboms" # All image SBOM targets (must match flake.nix packages) +# Format: name:license (SPDX identifier) SBOM_TARGETS=( - postgres - redis - minio - minio-client - sbomify-app - sbomify-keycloak - sbomify-caddy-dev - sbomify-minio-init + postgres:PostgreSQL + redis:AGPL-3.0-only + minio:AGPL-3.0-or-later + minio-client:Apache-2.0 + sbomify-app:Apache-2.0 + sbomify-keycloak:Apache-2.0 + sbomify-caddy-dev:Apache-2.0 + sbomify-minio-init:Apache-2.0 ) mkdir -p "$SBOM_DIR" @@ -26,7 +27,9 @@ echo "" failed=() -for target in "${SBOM_TARGETS[@]}"; do +for entry in "${SBOM_TARGETS[@]}"; do + target="${entry%%:*}" + license="${entry#*:}" echo "--- ${target} ---" sbom_attr=".#${target}-sbom" @@ -41,6 +44,21 @@ for target in "${SBOM_TARGETS[@]}"; do fi cp -L "$sbom_file" "${SBOM_DIR}/${target}.cdx.json" + + # Patch root component to describe the OCI image instead of the + # synthetic symlinkJoin closure name that bombon generates. + version=$(nix eval --raw nixpkgs#"${target//-/_}".version 2>/dev/null || echo "unknown") + image_name="wellmaintained/packages/${target}-image" + purl="pkg:docker/wellmaintained/packages/${target}@${version}" + + "${REPO_ROOT}/bin/patch-sbom-root" \ + --name "$image_name" \ + --version "$version" \ + --purl "$purl" \ + --license "$license" \ + < "${SBOM_DIR}/${target}.cdx.json" > "${SBOM_DIR}/${target}.cdx.json.tmp" \ + && mv "${SBOM_DIR}/${target}.cdx.json.tmp" "${SBOM_DIR}/${target}.cdx.json" + echo " -> ${SBOM_DIR}/${target}.cdx.json" # Quick verification diff --git a/bin/patch-sbom-root b/bin/patch-sbom-root new file mode 100755 index 0000000..1a55b5d --- /dev/null +++ b/bin/patch-sbom-root @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Patch the root component of a bombon-generated CycloneDX SBOM. +# +# bombon uses the symlinkJoin derivation name (e.g. "postgres-closure") as the +# root component, which carries no meaningful metadata. This script rewrites it +# to describe the OCI image with proper name, version, type, PURL, license, and +# wires up the dependency graph so the root depends on all closure components. +# +# Usage: patch-sbom-root --name NAME --version VER --purl PURL --license SPDX < in.cdx.json > out.cdx.json + +name="" version="" purl="" license="" + +while [ $# -gt 0 ]; do + case "$1" in + --name) name="$2"; shift 2 ;; + --version) version="$2"; shift 2 ;; + --purl) purl="$2"; shift 2 ;; + --license) license="$2"; shift 2 ;; + *) echo "Unknown argument: $1" >&2; exit 1 ;; + esac +done + +missing=() +[ -z "$name" ] && missing+=("--name") +[ -z "$version" ] && missing+=("--version") +[ -z "$purl" ] && missing+=("--purl") +[ -z "$license" ] && missing+=("--license") + +if [ ${#missing[@]} -gt 0 ]; then + echo "Missing required arguments: ${missing[*]}" >&2 + exit 1 +fi + +jq --arg name "$name" \ + --arg version "$version" \ + --arg purl "$purl" \ + --arg license "$license" \ + ' + .metadata.component.name = $name | + .metadata.component.version = $version | + .metadata.component.type = "container" | + .metadata.component.purl = $purl | + .metadata.component.licenses = [{"license": {"id": $license}}] | + .metadata.component["bom-ref"] as $root | + .dependencies = [ + {"ref": $root, "dependsOn": [.components[]?["bom-ref"]]} + ] + [.dependencies[]? | select(.ref != $root)] +' diff --git a/bin/sbom-compare b/bin/sbom-compare new file mode 100755 index 0000000..f27b6ad --- /dev/null +++ b/bin/sbom-compare @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Compare two SBOMs using sbomlyze and output structured JSON. +# +# Usage: sbom-compare --baseline FILE --current FILE --image NAME [--policy FILE] [--sbomlyze-cmd CMD] +# +# Runs sbomlyze in diff mode to produce markdown and JUnit outputs, +# and optionally checks a policy file for violations. +# Always exits 0 — policy violations are reported in the JSON output, +# not via exit code, so the caller can aggregate results before deciding. + +sbomlyze_cmd="sbomlyze" +baseline="" current="" image="" policy="" + +while [ $# -gt 0 ]; do + case "$1" in + --sbomlyze-cmd) sbomlyze_cmd="$2"; shift 2 ;; + --baseline) baseline="$2"; shift 2 ;; + --current) current="$2"; shift 2 ;; + --image) image="$2"; shift 2 ;; + --policy) policy="$2"; shift 2 ;; + -*) echo "Unknown option: $1" >&2; exit 1 ;; + *) echo "Unexpected argument: $1" >&2; exit 1 ;; + esac +done + +if [ -z "$baseline" ] || [ -z "$current" ] || [ -z "$image" ]; then + echo "Usage: sbom-compare --baseline FILE --current FILE --image NAME [--policy FILE] [--sbomlyze-cmd CMD]" >&2 + [ -z "$image" ] && echo "Missing required argument: --image" >&2 + exit 1 +fi + +[ ! -f "$baseline" ] && echo "Baseline file not found: $baseline" >&2 && exit 1 +[ ! -f "$current" ] && echo "Current file not found: $current" >&2 && exit 1 + +# Get markdown diff +diff_md=$("$sbomlyze_cmd" "$baseline" "$current" --format markdown 2>/dev/null || true) + +# Get JUnit XML +junit_xml=$("$sbomlyze_cmd" "$baseline" "$current" --format junit 2>/dev/null || true) + +# Check policy if provided +policy_pass=true +if [ -n "$policy" ]; then + if ! "$sbomlyze_cmd" "$baseline" "$current" --policy "$policy" > /dev/null 2>&1; then + policy_pass=false + fi +fi + +jq -n \ + --arg image "$image" \ + --arg diff_md "$diff_md" \ + --arg junit_xml "$junit_xml" \ + --argjson policy_pass "$policy_pass" \ + '{ + image: $image, + diff_md: $diff_md, + junit_xml: $junit_xml, + policy_pass: $policy_pass + }' diff --git a/bin/sbom-report b/bin/sbom-report new file mode 100755 index 0000000..d48df9e --- /dev/null +++ b/bin/sbom-report @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Generate a markdown PR comment from SBOM quality scores and diffs. +# +# Usage: sbom-report --scores-dir DIR +# +# Expects files in DIR matching: +# score-.json — output of bin/sbom-score (required) +# baseline-.json — output of bin/sbom-score for the baseline (optional) +# compare-.json — output of bin/sbom-compare (optional) +# +# Outputs markdown to stdout. +# Exits non-zero if any image's score regressed vs its baseline. + +scores_dir="" + +while [ $# -gt 0 ]; do + case "$1" in + --scores-dir) scores_dir="$2"; shift 2 ;; + -*) echo "Unknown option: $1" >&2; exit 1 ;; + *) echo "Unexpected argument: $1" >&2; exit 1 ;; + esac +done + +if [ -z "$scores_dir" ]; then + echo "Usage: sbom-report --scores-dir DIR" >&2 + exit 1 +fi + +regression=false + +# Collect all scored images +images=() +for f in "$scores_dir"/score-*.json; do + [ -f "$f" ] || continue + img=$(jq -r '.image' "$f") + images+=("$img") +done + +# Header +echo "## SBOM Quality Gate" +echo "" +echo "| Image | Score | Baseline | Delta | Status |" +echo "|-------|-------|----------|-------|--------|" + +# Table rows +for img in "${images[@]}"; do + score=$(jq -r '.score' "$scores_dir/score-${img}.json") + + if [ -f "$scores_dir/baseline-${img}.json" ]; then + baseline=$(jq -r '.score' "$scores_dir/baseline-${img}.json") + delta=$(printf '%.1f' "$(echo "$score - $baseline" | bc -l)") + + # Format delta with sign + if echo "$delta > 0" | bc -l | grep -q '^1$'; then + delta_fmt="+${delta}" + status="pass" + elif echo "$delta < 0" | bc -l | grep -q '^1$'; then + delta_fmt="${delta}" + status="regression" + regression=true + else + delta_fmt="0" + status="pass" + fi + + echo "| ${img} | ${score} | ${baseline} | ${delta_fmt} | ${status} |" + else + echo "| ${img} | ${score} | N/A | N/A | new baseline |" + fi +done + +# Diff details +for img in "${images[@]}"; do + if [ -f "$scores_dir/compare-${img}.json" ]; then + diff_md=$(jq -r '.diff_md // empty' "$scores_dir/compare-${img}.json") + if [ -n "$diff_md" ]; then + echo "" + echo "
" + echo "Diff: ${img}" + echo "" + echo "$diff_md" + echo "" + echo "
" + fi + fi +done + +if [ "$regression" = true ]; then + echo "SBOM quality regression detected" >&2 + exit 1 +fi diff --git a/bin/sbom-score b/bin/sbom-score new file mode 100755 index 0000000..43e7d4d --- /dev/null +++ b/bin/sbom-score @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Score an SBOM using sbomqs and output structured JSON. +# +# Usage: sbom-score [--sbomqs-cmd CMD] --image NAME +# +# Output (stdout): JSON with score, image name, component count, and category breakdown. + +sbomqs_cmd="sbomqs" +image="" + +while [ $# -gt 0 ]; do + case "$1" in + --sbomqs-cmd) sbomqs_cmd="$2"; shift 2 ;; + --image) image="$2"; shift 2 ;; + -*) echo "Unknown option: $1" >&2; exit 1 ;; + *) break ;; + esac +done + +if [ $# -eq 0 ]; then + echo "Usage: sbom-score [--sbomqs-cmd CMD] --image NAME " >&2 + exit 1 +fi + +sbom_file="$1" + +if [ ! -f "$sbom_file" ]; then + echo "SBOM file not found: $sbom_file" >&2 + exit 1 +fi + +# Run sbomqs and capture output +raw_output=$("$sbomqs_cmd" score "$sbom_file" --json 2>/dev/null) || { + echo "sbomqs failed to score $sbom_file" >&2 + exit 1 +} + +# Transform sbomqs output into our structured format +echo "$raw_output" | jq --arg image "$image" ' + .files[0] as $f | + { + image: $image, + score: $f.avg_score, + num_components: $f.num_components, + categories: [ + $f.scores[]? | {category: .category, score: .score, max_score: .max_score} + ] + } +' diff --git a/spec/patch_sbom_root_spec.sh b/spec/patch_sbom_root_spec.sh new file mode 100644 index 0000000..3975208 --- /dev/null +++ b/spec/patch_sbom_root_spec.sh @@ -0,0 +1,152 @@ +# shellcheck shell=sh +Describe "bin/patch-sbom-root" + setup() { + # Minimal bombon-like CycloneDX SBOM with a synthetic root component + SAMPLE_SBOM='{ + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "metadata": { + "component": { + "bom-ref": "pkg:nix/nixpkgs/postgres-closure", + "type": "application", + "name": "postgres-closure", + "version": "0" + } + }, + "components": [ + { + "bom-ref": "pkg:nix/nixpkgs/postgresql@17.4", + "type": "application", + "name": "postgresql", + "version": "17.4", + "licenses": [{"license": {"id": "PostgreSQL"}}] + }, + { + "bom-ref": "pkg:nix/nixpkgs/bash@5.2", + "type": "application", + "name": "bash", + "version": "5.2", + "licenses": [{"license": {"id": "GPL-3.0-or-later"}}] + } + ], + "dependencies": [] + }' + } + Before "setup" + + all_args="--name wellmaintained/packages/postgres-image --version 17.4 --purl pkg:docker/wellmaintained/packages/postgres@17.4 --license PostgreSQL" + + Describe "argument validation" + It "fails when --name is missing" + When run sh -c 'echo "$1" | bin/patch-sbom-root --version 17.4 --purl pkg:docker/x/p@1 --license PostgreSQL' _ "$SAMPLE_SBOM" + The status should be failure + The stderr should include "--name" + End + + It "fails when --version is missing" + When run sh -c 'echo "$1" | bin/patch-sbom-root --name x --purl pkg:docker/x/p@1 --license PostgreSQL' _ "$SAMPLE_SBOM" + The status should be failure + The stderr should include "--version" + End + + It "fails when --purl is missing" + When run sh -c 'echo "$1" | bin/patch-sbom-root --name x --version 1 --license PostgreSQL' _ "$SAMPLE_SBOM" + The status should be failure + The stderr should include "--purl" + End + + It "fails when --license is missing" + When run sh -c 'echo "$1" | bin/patch-sbom-root --name x --version 1 --purl pkg:docker/x/p@1' _ "$SAMPLE_SBOM" + The status should be failure + The stderr should include "--license" + End + End + + Describe "root component patching" + jq_query() { + echo "$SAMPLE_SBOM" | bin/patch-sbom-root $all_args | jq -r "$1" + } + + It "sets the root component name" + When call jq_query '.metadata.component.name' + The output should equal "wellmaintained/packages/postgres-image" + End + + It "sets the root component version" + When call jq_query '.metadata.component.version' + The output should equal "17.4" + End + + It "sets the root component type to container" + When call jq_query '.metadata.component.type' + The output should equal "container" + End + + It "sets the root component purl" + When call jq_query '.metadata.component.purl' + The output should equal "pkg:docker/wellmaintained/packages/postgres@17.4" + End + + It "sets the root component license" + When call jq_query '.metadata.component.licenses[0].license.id' + The output should equal "PostgreSQL" + End + + It "preserves the original bom-ref" + When call jq_query '.metadata.component["bom-ref"]' + The output should equal "pkg:nix/nixpkgs/postgres-closure" + End + End + + Describe "dependency graph wiring" + jq_query() { + echo "$SAMPLE_SBOM" | bin/patch-sbom-root $all_args | jq -r "$1" + } + + It "creates a dependency entry for the root component" + When call jq_query '.dependencies[0].ref' + The output should equal "pkg:nix/nixpkgs/postgres-closure" + End + + It "lists all components as dependsOn of root" + When call jq_query '.dependencies[0].dependsOn | length' + The output should equal 2 + End + + It "includes postgresql in dependsOn" + When call jq_query '.dependencies[0].dependsOn | sort | .[1]' + The output should equal "pkg:nix/nixpkgs/postgresql@17.4" + End + + It "includes bash in dependsOn" + When call jq_query '.dependencies[0].dependsOn | sort | .[0]' + The output should equal "pkg:nix/nixpkgs/bash@5.2" + End + End + + Describe "passthrough behavior" + jq_query() { + echo "$SAMPLE_SBOM" | bin/patch-sbom-root $all_args | jq -r "$1" + } + + It "preserves bomFormat" + When call jq_query '.bomFormat' + The output should equal "CycloneDX" + End + + It "preserves specVersion" + When call jq_query '.specVersion' + The output should equal "1.5" + End + + It "preserves all components" + When call jq_query '.components | length' + The output should equal 2 + End + + It "preserves component details" + When call jq_query '.components[0].name' + The output should equal "postgresql" + End + End +End diff --git a/spec/sbom_compare_spec.sh b/spec/sbom_compare_spec.sh new file mode 100644 index 0000000..cbb300b --- /dev/null +++ b/spec/sbom_compare_spec.sh @@ -0,0 +1,149 @@ +# shellcheck shell=sh +Describe "bin/sbom-compare" + setup() { + BASELINE="$(mktemp)" + CURRENT="$(mktemp)" + cat > "$BASELINE" <<'JSON' +{"bomFormat":"CycloneDX","specVersion":"1.5","components":[]} +JSON + cat > "$CURRENT" <<'JSON' +{"bomFormat":"CycloneDX","specVersion":"1.5","components":[]} +JSON + POLICY="$(mktemp)" + cat > "$POLICY" <<'JSON' +{"deny_licenses":["GPL-3.0-only"],"require_licenses":true} +JSON + } + cleanup() { + rm -f "$BASELINE" "$CURRENT" "$POLICY" + } + Before "setup" + After "cleanup" + + Describe "argument validation" + It "fails when no arguments are given" + When run bin/sbom-compare + The status should be failure + The stderr should include "Usage" + End + + It "fails when baseline file does not exist" + When run bin/sbom-compare --baseline /nonexistent --current "$CURRENT" --image postgres + The status should be failure + The stderr should include "not found" + End + + It "fails when current file does not exist" + When run bin/sbom-compare --baseline "$BASELINE" --current /nonexistent --image postgres + The status should be failure + The stderr should include "not found" + End + + It "fails when --image is missing" + When run bin/sbom-compare --baseline "$BASELINE" --current "$CURRENT" + The status should be failure + The stderr should include "--image" + End + End + + Describe "sbomlyze integration" + mock_sbomlyze() { + MOCK="$(mktemp)" + cat > "$MOCK" <<'SCRIPT' +#!/bin/sh +# Parse args to determine output format +for arg in "$@"; do + case "$arg" in + junit) cat <<'JUNIT' + + +JUNIT + exit 0 ;; + markdown) cat <<'MD' +## SBOM Diff +- 1 component added +- 0 components removed +MD + exit 0 ;; + esac +done +# Default: text output, exit 0 for no policy violations +exit 0 +SCRIPT + chmod +x "$MOCK" + echo "$MOCK" + } + + cleanup_mock() { + rm -f "$MOCK" + } + + It "outputs JSON with image name and diff results" + MOCK="$(mock_sbomlyze)" + When run bin/sbom-compare --sbomlyze-cmd "$MOCK" --baseline "$BASELINE" --current "$CURRENT" --image postgres + The status should be success + The output should include '"image"' + The output should include '"postgres"' + cleanup_mock + End + + jq_field() { + MOCK="$(mock_sbomlyze)" + bin/sbom-compare --sbomlyze-cmd "$MOCK" --baseline "$BASELINE" --current "$CURRENT" --image postgres | jq -r "$1" + cleanup_mock + } + + It "includes the markdown diff" + When call jq_field '.diff_md' + The output should include "SBOM Diff" + End + + It "includes the junit XML" + When call jq_field '.junit_xml' + The output should include "testsuites" + End + + It "reports policy_pass as true when no violations" + When call jq_field '.policy_pass' + The output should equal "true" + End + End + + Describe "policy violations" + mock_sbomlyze_fail() { + MOCK="$(mktemp)" + cat > "$MOCK" <<'SCRIPT' +#!/bin/sh +for arg in "$@"; do + case "$arg" in + junit) echo ''; exit 0 ;; + markdown) echo '## Diff'; exit 0 ;; + esac +done +# Policy check run: exit 1 = violations found +exit 1 +SCRIPT + chmod +x "$MOCK" + echo "$MOCK" + } + + jq_field_fail() { + MOCK="$(mock_sbomlyze_fail)" + bin/sbom-compare --sbomlyze-cmd "$MOCK" --baseline "$BASELINE" --current "$CURRENT" --image postgres --policy "$POLICY" | jq -r "$1" + rm -f "$MOCK" + } + + It "reports policy_pass as false when violations detected" + When call jq_field_fail '.policy_pass' + The output should equal "false" + End + + It "still succeeds (exit 0) even with policy violations" + MOCK="$(mock_sbomlyze_fail)" + When run bin/sbom-compare --sbomlyze-cmd "$MOCK" --baseline "$BASELINE" --current "$CURRENT" --image postgres --policy "$POLICY" + The status should be success + The output should include '"policy_pass"' + rm -f "$MOCK" + End + End +End diff --git a/spec/sbom_report_spec.sh b/spec/sbom_report_spec.sh new file mode 100644 index 0000000..90679c4 --- /dev/null +++ b/spec/sbom_report_spec.sh @@ -0,0 +1,112 @@ +# shellcheck shell=sh +Describe "bin/sbom-report" + setup() { + RESULTS_DIR="$(mktemp -d)" + + # Score result for postgres (improved) + cat > "$RESULTS_DIR/score-postgres.json" <<'JSON' +{"image":"postgres","score":7.2,"num_components":24,"categories":[{"category":"Licensing","score":6.5,"max_score":10.0}]} +JSON + + # Score result for redis (regressed) + cat > "$RESULTS_DIR/score-redis.json" <<'JSON' +{"image":"redis","score":6.0,"num_components":8,"categories":[{"category":"Licensing","score":5.0,"max_score":10.0}]} +JSON + + # Baseline scores + cat > "$RESULTS_DIR/baseline-postgres.json" <<'JSON' +{"image":"postgres","score":7.0,"num_components":24,"categories":[{"category":"Licensing","score":6.0,"max_score":10.0}]} +JSON + + cat > "$RESULTS_DIR/baseline-redis.json" <<'JSON' +{"image":"redis","score":6.5,"num_components":8,"categories":[{"category":"Licensing","score":5.5,"max_score":10.0}]} +JSON + + # Compare result for postgres + cat > "$RESULTS_DIR/compare-postgres.json" <<'JSON' +{"image":"postgres","diff_md":"1 component added","junit_xml":"","policy_pass":true} +JSON + + # Compare result for redis + cat > "$RESULTS_DIR/compare-redis.json" <<'JSON' +{"image":"redis","diff_md":"2 components removed","junit_xml":"","policy_pass":true} +JSON + } + cleanup() { + rm -rf "$RESULTS_DIR" + } + Before "setup" + After "cleanup" + + Describe "argument validation" + It "fails when --scores-dir is missing" + When run bin/sbom-report + The status should be failure + The stderr should include "Usage" + End + End + + Describe "markdown output" + It "produces a markdown table header" + When run bin/sbom-report --scores-dir "$RESULTS_DIR" + The output should include "SBOM Quality Gate" + The output should include "Image" + The output should include "Score" + The stderr should include "regression" + The status should be failure + End + + It "includes image scores" + When run bin/sbom-report --scores-dir "$RESULTS_DIR" + The output should include "postgres" + The output should include "7.2" + The output should include "redis" + The output should include "6.0" + The stderr should include "regression" + The status should be failure + End + + It "shows score delta when baselines exist" + When run bin/sbom-report --scores-dir "$RESULTS_DIR" + The output should include "+0.2" + The output should include "-0.5" + The stderr should include "regression" + The status should be failure + End + + It "includes diff details in collapsible sections" + When run bin/sbom-report --scores-dir "$RESULTS_DIR" + The output should include "
" + The output should include "1 component added" + The stderr should include "regression" + The status should be failure + End + End + + Describe "regression detection" + It "exits non-zero when any score regresses" + When run bin/sbom-report --scores-dir "$RESULTS_DIR" + The status should be failure + The stderr should include "regression" + The output should include "SBOM Quality Gate" + End + + It "exits zero when no regressions" + # Remove the regressed redis baseline so redis has no baseline + rm -f "$RESULTS_DIR/baseline-redis.json" + When run bin/sbom-report --scores-dir "$RESULTS_DIR" + The status should be success + The output should include "postgres" + End + End + + Describe "no baseline scenario" + It "shows N/A for images without baselines" + rm -f "$RESULTS_DIR/baseline-postgres.json" "$RESULTS_DIR/baseline-redis.json" + rm -f "$RESULTS_DIR/compare-postgres.json" "$RESULTS_DIR/compare-redis.json" + When run bin/sbom-report --scores-dir "$RESULTS_DIR" + The status should be success + The output should include "N/A" + End + End +End diff --git a/spec/sbom_score_spec.sh b/spec/sbom_score_spec.sh new file mode 100644 index 0000000..66ab12f --- /dev/null +++ b/spec/sbom_score_spec.sh @@ -0,0 +1,100 @@ +# shellcheck shell=sh +Describe "bin/sbom-score" + setup() { + SBOM_FILE="$(mktemp)" + cat > "$SBOM_FILE" <<'JSON' +{"bomFormat":"CycloneDX","specVersion":"1.5","components":[]} +JSON + } + cleanup() { + rm -f "$SBOM_FILE" + } + Before "setup" + After "cleanup" + + Describe "argument validation" + It "fails when no SBOM file is given" + When run bin/sbom-score + The status should be failure + The stderr should include "Usage" + End + + It "fails when SBOM file does not exist" + When run bin/sbom-score /nonexistent/file.json + The status should be failure + The stderr should include "not found" + End + End + + Describe "sbomqs integration" + # Mock sbomqs to avoid requiring the real binary in tests + mock_sbomqs() { + # Create a fake sbomqs that outputs realistic JSON + MOCK_SBOMQS="$(mktemp)" + cat > "$MOCK_SBOMQS" <<'SCRIPT' +#!/bin/sh +cat <<'MOCK' +{"files":[{"avg_score":7.2,"num_components":24,"scores":[{"category":"Licensing","score":6.5,"max_score":10.0},{"category":"Structural","score":8.1,"max_score":10.0},{"category":"Completeness","score":7.0,"max_score":10.0}]}]} +MOCK +SCRIPT + chmod +x "$MOCK_SBOMQS" + echo "$MOCK_SBOMQS" + } + + cleanup_mock() { + rm -f "$MOCK_SBOMQS" + } + + It "outputs valid JSON with score and image name" + MOCK_SBOMQS="$(mock_sbomqs)" + When run bin/sbom-score --sbomqs-cmd "$MOCK_SBOMQS" --image postgres "$SBOM_FILE" + The status should be success + The output should include '"image"' + The output should include '"score"' + cleanup_mock + End + + jq_field() { + MOCK_SBOMQS="$(mock_sbomqs)" + bin/sbom-score --sbomqs-cmd "$MOCK_SBOMQS" --image postgres "$SBOM_FILE" | jq -r "$1" + cleanup_mock + } + + It "extracts the average score" + When call jq_field '.score' + The output should equal "7.2" + End + + It "includes the image name" + When call jq_field '.image' + The output should equal "postgres" + End + + It "includes the component count" + When call jq_field '.num_components' + The output should equal "24" + End + + It "includes category scores" + When call jq_field '.categories | length' + The output should equal "3" + End + + It "includes Licensing category score" + When call jq_field '.categories[] | select(.category == "Licensing") | .score' + The output should equal "6.5" + End + End + + Describe "sbomqs failure handling" + It "fails when sbomqs returns an error" + MOCK_FAIL="$(mktemp)" + printf '#!/bin/sh\necho "error" >&2\nexit 1\n' > "$MOCK_FAIL" + chmod +x "$MOCK_FAIL" + When run bin/sbom-score --sbomqs-cmd "$MOCK_FAIL" --image postgres "$SBOM_FILE" + The status should be failure + The stderr should include "sbomqs" + rm -f "$MOCK_FAIL" + End + End +End diff --git a/spec/spec_helper.sh b/spec/spec_helper.sh new file mode 100644 index 0000000..80f6c91 --- /dev/null +++ b/spec/spec_helper.sh @@ -0,0 +1,13 @@ +# shellcheck shell=sh + +spec_helper_precheck() { + : minimum_version "0.28.1" +} + +spec_helper_loaded() { + : +} + +spec_helper_configure() { + : +}