From a0ed2bc25c430ffae8098561c68493cb9ecb5f44 Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Wed, 20 May 2026 20:33:38 +0300 Subject: [PATCH] ci/nightly: manifest aggregator + 90-build retention sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the two CI workflows that turn the dated nightly-* releases from PR #2111 into a queryable index served via GitHub Pages. manifest.yml — triggers on successful completion of build.yml (or by manual dispatch). Runs general/scripts/.../enrich_manifest.py to: - enumerate up to 200 GH releases, keep tags matching ^nightly-[0-9]{8}-[0-9a-f]{7}$, sort newest first, keep top 90; - for each, parse the body (sha=/short=/built_at= written by PR #2111) and the assets list, mapping openipc.--.tgz to platforms._.; - emit manifest.json (rich, JSON, for hosts/agents/CI) and manifest.flat (whitespace-delimited, for busybox sysupgrade); - commit both to the gh-pages branch. Resulting URLs: - https://openipc.github.io/firmware/manifest.json - https://openipc.github.io/firmware/manifest.flat cleanup.yml — Monday 05:00 UTC + workflow_dispatch. Deletes every dated nightly release beyond the 90 newest via `gh release delete --cleanup-tag`, then re-triggers manifest.yml to drop those entries from the index. A concurrency group `gh-pages-manifest` serializes the manifest and cleanup workflows so concurrent runs cannot race the gh-pages push. md5 is intentionally absent from the v1 schema — each .tgz already ships a .md5sum sidecar that sysupgrade validates after download, so a manifest-level md5 would be redundant and would require downloading every asset on each rebuild. The flat schema is therefore 5 columns: build_id platform flash size url. First-run behaviour: when no dated nightly-* releases exist yet, enrich_manifest writes an explicit empty manifest with a "first cron will populate" comment. Verified locally against the live repo. PR-B of six in the nightly-build redesign. Depends on PR #2111 (merged) for the dated release tag format. The gh-pages branch and Pages site are already configured. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/scripts/enrich_manifest.py | 141 +++++++++++++++++++++++++++++ .github/workflows/cleanup.yml | 44 +++++++++ .github/workflows/manifest.yml | 50 ++++++++++ 3 files changed, 235 insertions(+) create mode 100644 .github/scripts/enrich_manifest.py create mode 100644 .github/workflows/cleanup.yml create mode 100644 .github/workflows/manifest.yml diff --git a/.github/scripts/enrich_manifest.py b/.github/scripts/enrich_manifest.py new file mode 100644 index 0000000000..d1013530e2 --- /dev/null +++ b/.github/scripts/enrich_manifest.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Build manifest.json and manifest.flat for the nightly-* release index. + +Reads release metadata via the `gh` CLI (auth via $GH_TOKEN) and writes +two files into the directory given as the first argument: + + - manifest.json — full index for hosts, agents, CI tools + - manifest.flat — whitespace-delimited index for busybox-shell consumers + (on-device sysupgrade) + +md5 is intentionally omitted from the v1 schema. Each `.tgz` already +ships an in-archive `.md5sum` sidecar that sysupgrade validates after +download (`general/overlay/usr/sbin/sysupgrade:93-95`), so a manifest- +level md5 would be redundant and expensive to compute (would require +downloading every asset on each manifest rebuild). +""" +from __future__ import annotations + +import datetime as dt +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +REPO = os.environ.get("GITHUB_REPOSITORY", "OpenIPC/firmware") +RETENTION = 90 +TAG_RE = re.compile(r"^nightly-(\d{8})-([0-9a-f]{7})$") +ASSET_RE = re.compile(r"^openipc\.([^.]+)-(nor|nand)-(lite|ultimate|neo)\.tgz$") + + +def gh(*args: str) -> str: + return subprocess.check_output(["gh", *args], text=True) + + +def list_dated_releases() -> list[dict]: + raw = gh("release", "list", "--limit", "200", + "--json", "tagName,createdAt,isPrerelease") + rels = json.loads(raw) + dated = [r for r in rels if TAG_RE.match(r["tagName"])] + dated.sort(key=lambda r: r["createdAt"], reverse=True) + return dated[:RETENTION] + + +def fetch_release(tag: str) -> dict: + raw = gh("release", "view", tag, + "--json", "tagName,createdAt,body,assets") + return json.loads(raw) + + +def parse_body(body: str | None) -> tuple[str, str, str]: + sha = short = built_at = "" + for line in (body or "").splitlines(): + if line.startswith("sha="): + sha = line[4:].strip() + elif line.startswith("short="): + short = line[6:].strip() + elif line.startswith("built_at="): + built_at = line[9:].strip() + return sha, short, built_at + + +def parse_asset(name: str) -> tuple[str, str] | None: + m = ASSET_RE.match(name) + if not m: + return None + soc, flash, variant = m.groups() + return f"{soc}_{variant}", flash + + +def main() -> None: + out_dir = Path(sys.argv[1] if len(sys.argv) > 1 else ".") + out_dir.mkdir(parents=True, exist_ok=True) + + now = dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + dated = list_dated_releases() + + if not dated: + manifest = {"schema": 1, "generated_at": now, + "channels": {}, "builds": []} + (out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + (out_dir / "manifest.flat").write_text( + f"# generated_at={now}\n" + "# No nightly-YYYYMMDD- releases yet — " + "the first scheduled build will populate this index.\n" + ) + print("manifest: 0 builds (empty index)") + return + + builds = [] + for rel in dated: + info = fetch_release(rel["tagName"]) + sha, short, built_at = parse_body(info.get("body") or "") + platforms: dict[str, dict[str, dict]] = {} + for a in info.get("assets") or []: + parsed = parse_asset(a["name"]) + if not parsed: + continue + platform, flash = parsed + platforms.setdefault(platform, {})[flash] = { + "url": a["url"], + "size": a["size"], + } + builds.append({ + "id": info["tagName"], + "sha": sha, + "short": short, + "built_at": built_at or info["createdAt"], + "release_url": f"https://github.com/{REPO}/releases/tag/{info['tagName']}", + "platforms": platforms, + }) + + newest = builds[0]["id"] + manifest = { + "schema": 1, + "generated_at": now, + "channels": {"nightly": newest, "latest": newest}, + "builds": builds, + } + (out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + + lines = [ + "# OpenIPC firmware build index", + f"# generated_at={now}", + "# columns: build_id platform flash size url", + ] + for b in builds: + for platform, flashes in sorted(b["platforms"].items()): + for flash, info in sorted(flashes.items()): + lines.append(f"{b['id']} {platform} {flash} {info['size']} {info['url']}") + lines.append("# channels") + for ch, target in manifest["channels"].items(): + lines.append(f"@channel {ch} {target}") + (out_dir / "manifest.flat").write_text("\n".join(lines) + "\n") + + print(f"manifest: {len(builds)} builds, newest={newest}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml new file mode 100644 index 0000000000..3a12725a30 --- /dev/null +++ b/.github/workflows/cleanup.yml @@ -0,0 +1,44 @@ +name: cleanup +on: + schedule: + - cron: '0 5 * * 1' # Mondays 05:00 UTC + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: gh-pages-manifest + cancel-in-progress: false + +jobs: + prune: + name: Prune old nightly releases + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Delete releases beyond the 90 newest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + to_delete=$(gh release list --limit 200 --json tagName,createdAt \ + | jq -r '[ .[] | select(.tagName | test("^nightly-[0-9]{8}-[0-9a-f]{7}$")) ] + | sort_by(.createdAt) | reverse | .[90:] | .[].tagName') + + if [ -z "$to_delete" ]; then + echo "Nothing to delete; <=90 dated nightlies present." + exit 0 + fi + + echo "$to_delete" | while read -r tag; do + [ -z "$tag" ] && continue + echo "Deleting $tag" + gh release delete "$tag" --cleanup-tag --yes + done + + - name: Refresh manifest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh workflow run manifest.yml diff --git a/.github/workflows/manifest.yml b/.github/workflows/manifest.yml new file mode 100644 index 0000000000..68393562f2 --- /dev/null +++ b/.github/workflows/manifest.yml @@ -0,0 +1,50 @@ +name: manifest +on: + workflow_run: + workflows: [build] + types: [completed] + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: gh-pages-manifest + cancel-in-progress: false + +jobs: + generate: + name: Generate manifest + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + steps: + - name: Checkout master (for the script) + uses: actions/checkout@v4 + with: + path: master + + - name: Checkout gh-pages (for output) + uses: actions/checkout@v4 + with: + ref: gh-pages + path: pages + + - name: Generate manifest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + run: python3 master/.github/scripts/enrich_manifest.py pages + + - name: Publish to gh-pages + working-directory: pages + run: | + git config user.email "actions@github.com" + git config user.name "github-actions[bot]" + git add manifest.json manifest.flat + if git diff --cached --quiet; then + echo "No manifest changes; nothing to commit." + else + newest=$(jq -r '.channels.nightly // "empty"' manifest.json) + git commit -m "manifest: $(date -u +%FT%TZ) — ${newest}" + git push + fi