From cd997dbfc28e4cf30a22c42ec90742bf84af446f Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Thu, 21 May 2026 07:49:07 +0300 Subject: [PATCH 1/2] ci/manifest: publish on partial-failure builds; fix empty built_at= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Last night's cron (run 26196546684) had one matrix job fail — hi3516av100_ultimate hit transient 502/timeouts pulling the toolchain and exhausted its retry budget. The 91 other boards uploaded their artifacts successfully and the dated release `nightly-20260520-887328c` ended with 102 assets, the rolling `nightly` was a faithful mirror, and the SHA-gate worked correctly. But manifest.yml's trigger condition was github.event.workflow_run.conclusion == 'success' which evaluates the whole workflow's conclusion (failure if any matrix job failed). The manifest workflow was therefore SKIPPED for last night's build and gh-pages still served the empty-state placeholder even though 102 valid artifacts existed. Fix #1: trigger manifest on success OR failure, only skip on 'cancelled'. enrich_manifest.py reads whatever assets actually exist on the release, so a partial release just shows up in the index with fewer platforms in its `platforms` map — the right behaviour. Fix #2: built_at= rendered empty in the release body because ${{ github.run_started_at }} doesn't interpolate inside softprops/ action-gh-release@v2's `body:` field (likely a context-availability quirk). Compute BUILT_AT in the preflight job with `date -u +...`, expose it as a job output, reference it from both upload steps. Now the body has a real RFC3339 timestamp. Manually backfilled tonight's stale manifest after the issue was identified — the workflow_dispatch path was already correct, and manifest.flat now serves 102 entries for nightly-20260520-887328c. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/build.yml | 7 +++++-- .github/workflows/manifest.yml | 9 ++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aea5e1acf..eb78176fc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,6 +16,7 @@ jobs: head_sha: ${{ steps.gate.outputs.head_sha }} short_sha: ${{ steps.gate.outputs.short_sha }} build_id: ${{ steps.gate.outputs.build_id }} + built_at: ${{ steps.gate.outputs.built_at }} steps: - uses: actions/checkout@v4 - id: gate @@ -25,6 +26,7 @@ jobs: HEAD=$(git rev-parse HEAD) SHORT=$(git rev-parse --short HEAD) BUILD_ID="nightly-$(date -u +%Y%m%d)-${SHORT}" + BUILT_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ) PREV=$(gh release view nightly --json body -q .body 2>/dev/null \ | sed -n 's/^sha=//p' | head -1 || true) if [ "${{ github.event_name }}" = "schedule" ] && [ "$PREV" = "$HEAD" ]; then @@ -37,6 +39,7 @@ jobs: echo "head_sha=$HEAD" >> "$GITHUB_OUTPUT" echo "short_sha=$SHORT" >> "$GITHUB_OUTPUT" echo "build_id=$BUILD_ID" >> "$GITHUB_OUTPUT" + echo "built_at=$BUILT_AT" >> "$GITHUB_OUTPUT" buildroot: name: Firmware @@ -285,7 +288,7 @@ jobs: body: | sha=${{ needs.preflight.outputs.head_sha }} short=${{ needs.preflight.outputs.short_sha }} - built_at=${{ github.run_started_at }} + built_at=${{ needs.preflight.outputs.built_at }} files: | ${{env.NORFW}} ${{env.NANDFW}} @@ -298,7 +301,7 @@ jobs: body: | sha=${{ needs.preflight.outputs.head_sha }} short=${{ needs.preflight.outputs.short_sha }} - built_at=${{ github.run_started_at }} + built_at=${{ needs.preflight.outputs.built_at }} files: | ${{env.NORFW}} ${{env.NANDFW}} diff --git a/.github/workflows/manifest.yml b/.github/workflows/manifest.yml index 68393562f..ea6b07ba5 100644 --- a/.github/workflows/manifest.yml +++ b/.github/workflows/manifest.yml @@ -15,7 +15,14 @@ concurrency: jobs: generate: name: Generate manifest - if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + # Publish on success AND on partial-failure: one flaky board (e.g. a + # transient toolchain 502) should not deny manifest updates for the 90+ + # platforms that did upload artifacts. enrich_manifest.py reads whatever + # assets actually exist on the release, so a partial release just shows + # up with fewer platforms in its `platforms` map. + if: >- + github.event_name == 'workflow_dispatch' || + contains(fromJSON('["success", "failure"]'), github.event.workflow_run.conclusion) runs-on: ubuntu-latest steps: - name: Checkout master (for the script) From 55d77dcd3719f8d3bd2a641432a9507e0c674740 Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Thu, 21 May 2026 08:01:06 +0300 Subject: [PATCH 2/2] ci/build: extend retry budget for upstream toolchain/CDN flakes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add two more entries to the per-board retry backoff: 30 60 120 300 -> 30 60 120 300 600 1200. Total attempts: 5 -> 7. Max idle sleep: ~8.5 min -> ~40 min. Motivated by hi3516av100_ultimate failing on the 2026-05-20 cron (run 26196546684): GitHub releases CDN / toolchain mirror was returning 502 Bad Gateway over a >10 min window, and the existing budget exhausted before it cleared. The build itself was fine; just upstream flake. The longer tail only fires on persistent upstream issues — happy-path builds still finish in their first attempt with zero added wait. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/build.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eb78176fc..be50a4342 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -250,7 +250,11 @@ jobs: mkdir -p /tmp/ccache ln -s /tmp/ccache ${HOME}/.ccache - backoffs="30 60 120 300" + # Backoffs give 7 attempts total. The longer tail (600, 1200) covers + # the GitHub releases CDN / toolchain mirror flakes that took down + # hi3516av100_ultimate on 2026-05-20 (run 26196546684) — 502s + # storming for >10 min outlasted the previous 30/60/120/300 budget. + backoffs="30 60 120 300 600 1200" attempt=1 for sleep_for in $backoffs ""; do make BOARD=${{matrix.platform}} && break