Build images natively per architecture

Replace the QEMU-based multi-platform build with native amd64 and arm64 runner jobs, then publish the final image tags by assembling a manifest from the architecture-specific tags. Fixes nikolaik#258.

Run the smoke test suite against each architecture-specific image before publishing the final manifest, instead of only testing the locally loaded amd64 image. The build-matrix helper now emits an architecture-expanded matrix for the workflow, and the new unit test covers that expansion. Fixes nikolaik#314.

Author: Wuodan

.github/workflows/build.yaml

@@ -22,7 +22,8 @@ jobs:
     runs-on: ubuntu-latest
     needs: [test]
     outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      version_matrix: ${{ steps.set-matrix.outputs.matrix }}
+      arch_matrix: ${{ steps.set-matrix.outputs.arch_matrix }}
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
         with:
@@ -37,23 +38,24 @@ jobs:
           uv run dpn $FORCE build-matrix --event ${{ github.event_name }}
 
 
-  deploy:
-    name: ${{ matrix.key }}
-    runs-on: ubuntu-latest
-    if: needs.generate-matrix.outputs.matrix != ''
+  build-arch:
+    name: ${{ matrix.key }} (${{ matrix.arch }})
+    runs-on: ${{ matrix.runner }}
+    if: needs.generate-matrix.outputs.arch_matrix != ''
     needs: [generate-matrix]
     strategy:
-      matrix: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }}
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.generate-matrix.outputs.arch_matrix) }}
     steps:
       # Setup
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
       - uses: astral-sh/setup-uv@e06108dd0aef18192324c70427afc47652e63a82 # v7
         with:
           enable-cache: true
       - name: Generate Dockerfile from config
-        run: uv run dpn dockerfile --context '${{ toJSON(matrix) }}'
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4
+        run: |
+          context="$(echo '${{ toJSON(matrix) }}' | jq -c '{key, python, python_canonical, python_image, nodejs, nodejs_canonical, distro, platforms, digest}')"
+          uv run dpn dockerfile --context "${context}"
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4
       - name: Login to Docker Hub
@@ -62,36 +64,51 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      # Build
+      # Build and push
       - name: Build image
+        id: build-and-push
         uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
         with:
           context: .
           file: dockerfiles/${{ matrix.key }}.Dockerfile
-          load: true
-          tags: ${{ env.IMAGE_NAME }}:${{ matrix.key }}
+          platforms: ${{ matrix.platform }}
+          push: true
+          tags: ${{ env.IMAGE_NAME }}:${{ matrix.key }}-${{ matrix.arch }}
 
       # Test
       - name: Run smoke tests
         run: |
-          docker run --rm ${{ env.IMAGE_NAME }}:${{ matrix.key }} sh -c "node --version && npm --version && yarn --version && python --version && pip --version && pipenv --version && poetry --version && uv --version"
+          docker run --rm ${{ env.IMAGE_NAME }}@${{ steps.build-and-push.outputs.digest }} sh -c "node --version && npm --version && yarn --version && python --version && pip --version && pipenv --version && poetry --version && uv --version"
 
-      # Push image
-      - name: Push image
-        uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
-        id: build-and-push
+  deploy:
+    name: Publish ${{ matrix.key }}
+    runs-on: ubuntu-latest
+    if: needs.generate-matrix.outputs.version_matrix != ''
+    needs: [generate-matrix, build-arch]
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.generate-matrix.outputs.version_matrix) }}
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4
+      - name: Login to Docker Hub
+        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4
         with:
-          context: .
-          file: dockerfiles/${{ matrix.key }}.Dockerfile
-          platforms: ${{ join(matrix.platforms) }}
-          push: true
-          tags: ${{ env.IMAGE_NAME }}:${{ matrix.key }}
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Publish multi-arch manifest
+        run: |
+          tags=("${IMAGE_NAME}:${{ matrix.key }}-amd64")
+          if echo '${{ toJSON(matrix.platforms) }}' | jq -e '.[] == "linux/arm64"' > /dev/null; then
+            tags+=("${IMAGE_NAME}:${{ matrix.key }}-arm64")
+          fi
+          docker buildx imagetools create --tag "${IMAGE_NAME}:${{ matrix.key }}" "${tags[@]}"
 
-      # Store build context
       - name: Add digest to build context
         run: |
           mkdir builds/
-          digest="${{ steps.build-and-push.outputs.digest }}"
+          digest="$(docker buildx imagetools inspect "${IMAGE_NAME}:${{ matrix.key }}" | awk '/^Digest:/ {print $2}')"
           echo '${{ toJSON(matrix) }}' | jq --arg digest "$digest" '. +={"digest": $digest}' >> "builds/${{ matrix.key }}.json"
 
       - name: Upload build context

src/docker_python_nodejs/build_matrix.py

@@ -25,15 +25,42 @@ def _github_action_set_output(key: str, value: str) -> None:
         sys.exit(1)
 
     with Path(GITHUB_OUTPUT).open("a") as fp:
-        fp.write(f"{key}={value}")
+        fp.write(f"{key}={value}\n")
+
+
+def _build_matrix_json(new_or_updated: list[BuildVersion]) -> str:
+    return json.dumps({"include": [dataclasses.asdict(ver) for ver in new_or_updated]}) if new_or_updated else ""
+
+
+def _build_arch_matrix_json(new_or_updated: list[BuildVersion]) -> str:
+    if not new_or_updated:
+        return ""
+
+    include: list[dict[str, object]] = []
+    for version in new_or_updated:
+        include.extend(
+            (
+                dataclasses.asdict(version)
+                | {
+                    "platform": platform,
+                    "arch": platform.split("/")[1],
+                    "runner": "ubuntu-24.04-arm" if platform == "linux/arm64" else "ubuntu-latest",
+                }
+            )
+            for platform in version.platforms
+        )
+
+    return json.dumps({"include": include})
 
 
 def build_matrix(new_or_updated: list[BuildVersion], ci_event: str) -> None:
     if not new_or_updated and ci_event == CI_EVENT_SCHEDULED:
         logger.info("\n# Scheduled run with no new or updated versions. Doing nothing.")
         return
 
-    matrix = json.dumps({"include": [dataclasses.asdict(ver) for ver in new_or_updated]}) if new_or_updated else ""
-    _github_action_set_output("MATRIX", matrix)
+    matrix = _build_matrix_json(new_or_updated)
+    arch_matrix = _build_arch_matrix_json(new_or_updated)
+    _github_action_set_output("matrix", matrix)
+    _github_action_set_output("arch_matrix", arch_matrix)
     logger.info("\n# New or updated versions:")
     logger.info("Nothing" if not new_or_updated else "\n".join(version.key for version in new_or_updated))

tests/test_all.py

@@ -8,6 +8,7 @@
 import pytest
 import responses
 
+from docker_python_nodejs.build_matrix import _build_arch_matrix_json
 from docker_python_nodejs.dockerfiles import render_dockerfile_with_context
 from docker_python_nodejs.readme import update_dynamic_readme
 from docker_python_nodejs.settings import BASE_PATH, DOCKERFILES_PATH
@@ -292,6 +293,43 @@ def test_find_new_or_updated_with_digest() -> None:
     assert len(res) == 0
 
 
+def test_build_arch_matrix_json(build_version: BuildVersion) -> None:
+    matrix = json.loads(_build_arch_matrix_json([build_version]))
+
+    assert matrix == {
+        "include": [
+            {
+                "key": "python3.11-nodejs20",
+                "python": "3.11",
+                "python_canonical": "3.11.3",
+                "python_image": "3.11.3-trixie",
+                "nodejs": "20",
+                "nodejs_canonical": "20.2.0",
+                "distro": "trixie",
+                "platforms": ["linux/amd64", "linux/arm64"],
+                "digest": "",
+                "platform": "linux/amd64",
+                "arch": "amd64",
+                "runner": "ubuntu-latest",
+            },
+            {
+                "key": "python3.11-nodejs20",
+                "python": "3.11",
+                "python_canonical": "3.11.3",
+                "python_image": "3.11.3-trixie",
+                "nodejs": "20",
+                "nodejs_canonical": "20.2.0",
+                "distro": "trixie",
+                "platforms": ["linux/amd64", "linux/arm64"],
+                "digest": "",
+                "platform": "linux/arm64",
+                "arch": "arm64",
+                "runner": "ubuntu-24.04-arm",
+            },
+        ],
+    }
+
+
 @responses.activate
 def test_latest_tag_key_matches_legacy_latest_sources() -> None:
     responses.add(