Precompiled: Ubuntu26.04 driver container support

.common-ci.yml

@@ -110,6 +110,14 @@ trigger-pipeline:
         KERNEL_FLAVOR: [aws, azure, azure-fde, generic, nvidia, oracle]
         LTS_KERNEL: ["6.8"]
 
+# Define the matrix of precompiled jobs that can be run in parallel for ubuntu26.04
+.driver-versions-precompiled-ubuntu26.04:
+  parallel:
+    matrix:
+      - DRIVER_BRANCH: [580]
+        KERNEL_FLAVOR: [aws, azure, azure-fde, generic, nvidia, oracle]
+        LTS_KERNEL: ["6.14"]
+
 .dist-ubuntu22.04:
   variables:
     DIST: ubuntu22.04
@@ -467,3 +475,14 @@ release:staging-precompiled-ubuntu24.04:
     - .release:staging-precompiled
   needs:
     - image-precompiled-ubuntu24.04
+
+# Precompiled Ubuntu26.04 release
+release:staging-precompiled-ubuntu26.04:
+  variables:
+    DIST: signed_ubuntu26.04
+    BASE_TARGET: questing
+  extends:
+    - .driver-versions-precompiled-ubuntu26.04
+    - .release:staging-precompiled
+  needs:
+    - image-precompiled-ubuntu26.04

.github/workflows/precompiled.yaml

@@ -29,6 +29,7 @@ jobs:
     runs-on: linux-amd64-cpu4
     outputs:
       driver_branch: ${{ steps.extract_driver_branch.outputs.driver_branch }}
+      exclude_build_matrix_pairs: ${{ steps.extract_driver_branch.outputs.exclude_build_matrix_pairs }}
       kernel_flavors: ${{ steps.extract_driver_branch.outputs.kernel_flavors }}
       dist: ${{ steps.extract_driver_branch.outputs.dist }}
       lts_kernel: ${{ steps.extract_driver_branch.outputs.lts_kernel }}
@@ -39,7 +40,7 @@ jobs:
         id: extract_driver_branch
         run: |
           # get driver_branch
-          DRIVER_BRANCH=("535" "580")
+          DRIVER_BRANCH=("535" "580" "595")
           driver_branch_json=$(printf '%s\n' "${DRIVER_BRANCH[@]}" | jq -R . | jq -cs .)
           echo "driver_branch=$driver_branch_json" >> $GITHUB_OUTPUT
 
@@ -49,15 +50,18 @@ jobs:
           echo "kernel_flavors=$kernel_flavors_json" >> $GITHUB_OUTPUT
 
           # get ubuntu distributions
-          DIST=("ubuntu22.04" "ubuntu24.04")
+          DIST=("ubuntu22.04" "ubuntu24.04" "ubuntu26.04")
           dist_json=$(printf '%s\n' "${DIST[@]}" | jq -R . | jq -cs .)
           echo "dist=$dist_json" >> $GITHUB_OUTPUT
 
           # LTS_KERNEL setup
-          LTS_KERNEL=("5.15" "6.8")
+          LTS_KERNEL=("5.15" "6.8" "7.0")
           lts_kernel_json=$(printf '%s\n' "${LTS_KERNEL[@]}" | jq -R . | jq -cs .)
           echo "lts_kernel=$lts_kernel_json" >> $GITHUB_OUTPUT
 
+          EXCLUDE_BUILD_MATRIX_PAIRS=("ubuntu22.04 595" "ubuntu24.04 535" "ubuntu26.04 535" "ubuntu26.04 580")
+          echo "exclude_build_matrix_pairs=$(printf '%s\n' "${EXCLUDE_BUILD_MATRIX_PAIRS[@]}" | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT
+
   precompiled-build-image:
     needs: set-driver-version-matrix
     runs-on: linux-amd64-cpu4
@@ -70,10 +74,25 @@ jobs:
         exclude:
           - dist: ubuntu24.04
             driver_branch: 535
+          - dist: ubuntu26.04
+            driver_branch: 535
+          - dist: ubuntu26.04
+            driver_branch: 580
+          - dist: ubuntu22.04
+            driver_branch: 595
           - lts_kernel: 5.15
             dist: ubuntu24.04
+          - lts_kernel: 7.0
+            dist: ubuntu24.04
+          - lts_kernel: 5.15
+            dist: ubuntu26.04
+          - lts_kernel: 6.8
+            dist: ubuntu26.04
+          - lts_kernel: 7.0
+            dist: ubuntu22.04
           - flavor: azure-fde
             dist: ubuntu22.04
+      max-parallel: 25
     steps:
       - uses: actions/checkout@v6
         name: Check out code
@@ -113,6 +132,8 @@ jobs:
             BASE_TARGET="jammy"
           elif [[ "${{ matrix.dist }}" == "ubuntu24.04" ]]; then
             BASE_TARGET="noble"
+          elif [[ "${{ matrix.dist }}" == "ubuntu26.04" ]]; then
+            BASE_TARGET="resolute"
           fi
           make DRIVER_BRANCH=${{ matrix.driver_branch }} KERNEL_FLAVOR=${{ matrix.flavor }} LTS_KERNEL=${LTS_KERNEL} build-base-${BASE_TARGET}
 
@@ -143,6 +164,8 @@ jobs:
             BASE_TARGET="jammy"
           elif [[ "${{ matrix.dist }}" == "ubuntu24.04" ]]; then
             BASE_TARGET="noble"
+          elif [[ "${{ matrix.dist }}" == "ubuntu26.04" ]]; then
+            BASE_TARGET="resolute"
           fi
           tar -cvf kernel-version-${{ matrix.driver_branch }}-${KERNEL_VERSION}-${DIST}.tar kernel_version.txt
           docker save "${PRIVATE_REGISTRY}/nvidia/driver:base-${BASE_TARGET}-${LTS_KERNEL}-${{ matrix.flavor }}-${{ matrix.driver_branch }}" \
@@ -183,6 +206,14 @@ jobs:
         exclude:
           - lts_kernel: 5.15
             dist: ubuntu24.04
+          - lts_kernel: 7.0
+            dist: ubuntu24.04
+          - lts_kernel: 5.15
+            dist: ubuntu26.04
+          - lts_kernel: 6.8
+            dist: ubuntu26.04
+          - lts_kernel: 7.0
+            dist: ubuntu22.04
     needs:
       - precompiled-build-image
       - set-driver-version-matrix
@@ -212,14 +243,14 @@ jobs:
           kernel_flavors_json='${{ needs.set-driver-version-matrix.outputs.kernel_flavors }}'
           KERNEL_FLAVORS=($(echo "$kernel_flavors_json" | jq -r '.[]'))
           driver_branch_json='${{ needs.set-driver-version-matrix.outputs.driver_branch }}'
-          DRIVER_BRANCHES=($(echo "$driver_branch_json" | jq -r '.[]'))
-
-          # remove 535 driver branch for ubuntu24.04
-          if [ "$DIST" == "ubuntu24.04" ]; then
-            DRIVER_BRANCHES=($(for branch in "${DRIVER_BRANCHES[@]}"; do
-            [[ $branch != "535" ]] && echo "$branch"
-          done))
-          fi
+          exclude_pairs_json='${{ needs.set-driver-version-matrix.outputs.exclude_build_matrix_pairs }}'
+          DRIVER_BRANCHES=()
+          for b in $(echo "$driver_branch_json" | jq -r '.[]'); do
+            pair="$DIST $b"
+            if ! echo "$exclude_pairs_json" | jq -r '.[]' | grep -qx "$pair"; then
+              DRIVER_BRANCHES+=("$b")
+            fi
+          done
           source ./tests/scripts/ci-precompiled-helpers.sh
           KERNEL_VERSIONS=($(get_kernel_versions_to_test KERNEL_FLAVORS[@] DRIVER_BRANCHES[@] $DIST $LTS_KERNEL))
           if [ -z "$KERNEL_VERSIONS" ]; then
@@ -358,18 +389,20 @@ jobs:
           echo "DIST=$DIST" >> $GITHUB_ENV
           KERNEL_VERSION=${KERNEL_VERSION%-*}
           echo "KERNEL_VERSION=$KERNEL_VERSION" >> $GITHUB_ENV
-          driver_branch_json="${{ needs.set-driver-version-matrix.outputs.driver_branch }}"
-          DRIVER_BRANCHES=($(echo "$driver_branch_json" | jq -r '.[]'))
-          echo "DRIVER_BRANCHES=${DRIVER_BRANCHES[*]}" >> $GITHUB_ENV
+          printf 'DRIVER_BRANCHES_JSON=%s\n' '${{ needs.set-driver-version-matrix.outputs.driver_branch }}' >> $GITHUB_ENV
+          printf 'EXCLUDE_PAIRS_JSON=%s\n' '${{ needs.set-driver-version-matrix.outputs.exclude_build_matrix_pairs }}' >> $GITHUB_ENV
       - name: Configure Holodeck e2e test config (kernel, instance)
         run: |
           yq eval '.spec += {"kernel": {"version": strenv(KERNEL_VERSION)}}' -i tests/holodeck_ubuntu.yaml
           if [[ "$DIST" == "ubuntu24.04" ]]; then
             yq eval '.spec.instance.os = "ubuntu-24.04"' -i tests/holodeck_ubuntu.yaml
           fi
+          if [[ "$DIST" == "ubuntu26.04" ]]; then
+            yq eval '.spec.instance.os = "ubuntu-26.04"' -i tests/holodeck_ubuntu.yaml
+          fi
 
       - name: Set up Holodeck
-        uses: NVIDIA/holodeck@v0.3.2
+        uses: NVIDIA/holodeck@ubuntu26.04
         env:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -401,15 +434,13 @@ jobs:
           GH_TOKEN:  ${{ secrets.GITHUB_TOKEN }}
         run: |
           rc=0
-          # for precompiled driver we are setting driver branch as driver version
-          DRIVER_BRANCHES=(${{ env.DRIVER_BRANCHES }})
-          # remove 535 driver branch for ubuntu24.04
-          if [ "$DIST" == "ubuntu24.04" ]; then
-            DRIVER_BRANCHES=($(for branch in "${DRIVER_BRANCHES[@]}"; do
-            [[ $branch != "535" ]] && echo "$branch"
-          done))
-          fi
+          DRIVER_BRANCHES=($(echo '${{ env.DRIVER_BRANCHES_JSON }}' | jq -r '.[]'))
+          exclude_pairs='${{ env.EXCLUDE_PAIRS_JSON }}'
           for DRIVER_VERSION in "${DRIVER_BRANCHES[@]}"; do
+            pair="$DIST $DRIVER_VERSION"
+            if echo "$exclude_pairs" | jq -r '.[]' | grep -qx "$pair"; then
+              continue
+            fi
             echo "Running e2e for DRIVER_VERSION=$DRIVER_VERSION"
             status=0
             TEST_CASE_ARGS="${GPU_OPERATOR_OPTIONS} --set driver.version=${DRIVER_VERSION}"
@@ -455,18 +486,26 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name:  Set image vars
+        id: set_image_vars
         run: |
           echo "PRIVATE_REGISTRY=ghcr.io" >> $GITHUB_ENV
+          KERNEL_VERSION="${{ matrix.kernel_version }}"
+          DIST="${KERNEL_VERSION##*-}"
+          pair="$DIST ${{ matrix.driver_branch }}"
+          echo "run_publish=true" >> $GITHUB_OUTPUT
+          if echo '${{ needs.set-driver-version-matrix.outputs.exclude_build_matrix_pairs }}' | jq -r '.[]' | grep -qx "$pair"; then
+            echo "run_publish=false" >> $GITHUB_OUTPUT
+          fi
 
       - name: Download base image artifact
-        if: ${{ ! (matrix.driver_branch == 535 && contains(matrix.kernel_version, 'ubuntu24.04')) }}
+        if: steps.set_image_vars.outputs.run_publish == 'true'
         uses: actions/download-artifact@v8
         with:
           name: base-images-${{ matrix.driver_branch }}-${{ matrix.kernel_version }}
           path: ./
 
       - name: Publish base image
-        if: ${{ ! (matrix.driver_branch == 535 && contains(matrix.kernel_version, 'ubuntu24.04')) }}
+        if: steps.set_image_vars.outputs.run_publish == 'true'
         run: |
           LTS_KERNEL=$(echo "${{ matrix.kernel_version }}" | sed -E 's/^([0-9]+\.[0-9]+)\..*/\1/')
           KERNEL_FLAVOR=$(echo "${{ matrix.kernel_version }}" | sed -E 's/^[0-9]+\.[0-9]+\.[0-9]+-[0-9]+-(.*)-ubuntu[0-9]+\.[0-9]+$/\1/')
@@ -475,6 +514,8 @@ jobs:
             BASE_TARGET="jammy"
           elif [[ "${DIST}" == "ubuntu24.04" ]]; then
             BASE_TARGET="noble"
+          elif [[ "${DIST}" == "ubuntu26.04" ]]; then
+            BASE_TARGET="resolute"
           fi
           image_path="./base-images-${{ matrix.driver_branch }}-${{ matrix.kernel_version }}.tar"
           echo "uploading  $image_path"
@@ -486,14 +527,14 @@ jobs:
           fi
 
       - name: Download built image artifact
-        if: ${{ ! (matrix.driver_branch == 535 && contains(matrix.kernel_version, 'ubuntu24.04')) }}
+        if: steps.set_image_vars.outputs.run_publish == 'true'
         uses: actions/download-artifact@v8
         with:
            name: driver-images-${{ matrix.driver_branch }}-${{ matrix.kernel_version }}
            path: ./
 
       - name: Publish image
-        if: ${{ ! (matrix.driver_branch == 535 && contains(matrix.kernel_version, 'ubuntu24.04')) }}
+        if: steps.set_image_vars.outputs.run_publish == 'true'
         run: |
           image_path="./driver-images-${{ matrix.driver_branch }}-${{ matrix.kernel_version }}.tar"
           echo "uploading  $image_path"
@@ -505,7 +546,7 @@ jobs:
           fi
 
       - name: Slack notification
-        if: ${{ ! (matrix.driver_branch == 535 && contains(matrix.kernel_version, 'ubuntu24.04')) && github.ref == 'refs/heads/main' }}
+        if: ${{ steps.set_image_vars.outputs.run_publish == 'true' && github.ref == 'refs/heads/main' }}
         uses: slackapi/slack-github-action@v3.0.3
         with:
           token: ${{ secrets.SLACK_BOT_TOKEN }}

.gitlab-ci.yml

@@ -137,3 +137,12 @@ image-precompiled-ubuntu24.04:
   extends:
     - .driver-versions-precompiled-ubuntu24.04
     - .image-build-precompiled
+
+image-precompiled-ubuntu26.04:
+  variables:
+    DIST: signed_ubuntu26.04
+    BASE_TARGET: questing
+    CVE_UPDATES: "curl libc6"
+  extends:
+    - .driver-versions-precompiled-ubuntu26.04
+    - .image-build-precompiled

.nvidia-ci.yml

@@ -112,6 +112,20 @@ image-precompiled-ubuntu24.04:
     - .driver-versions-precompiled-ubuntu24.04
     - .image-pull-generic
 
+image-precompiled-ubuntu26.04:
+  variables:
+    DIST: signed_ubuntu26.04
+    BASE_TARGET: questing
+    PRECOMPILED: "true"
+    CVE_UPDATES: "curl libc6"
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: delayed
+      start_in: 30 minutes
+  extends:
+    - .driver-versions-precompiled-ubuntu26.04
+    - .image-pull-generic
+
 .image-pull-ubuntu22.04:
   # Perform for each DRIVER_VERSION
   extends:
@@ -281,6 +295,18 @@ image-rocky10:
     - !reference [.scan-rules-common, rules]
     - !reference [.precompiled-rules, rules]
 
+.scan-precompiled-ubuntu26.04:
+  variables:
+    DIST: signed_ubuntu26.04
+    BASE_TARGET: questing
+    PRECOMPILED: "true"
+  extends:
+    - .driver-versions-precompiled-ubuntu26.04
+    - .scan-generic
+  rules:
+    - !reference [.scan-rules-common, rules]
+    - !reference [.precompiled-rules, rules]
+
 .scan-precompiled-ubuntu22.04:
   variables:
     DIST: signed_ubuntu22.04
@@ -334,6 +360,15 @@ scan-precompiled-ubuntu24.04-amd64:
   needs:
     - image-precompiled-ubuntu24.04
 
+scan-precompiled-ubuntu26.04-amd64:
+  variables:
+    PLATFORM: linux/amd64
+  extends:
+    - .scan-precompiled-ubuntu26.04
+    - .platform-amd64
+  needs:
+    - image-precompiled-ubuntu26.04
+
 scan-precompiled-ubuntu22.04-amd64:
   variables:
     PLATFORM: linux/amd64
@@ -476,6 +511,18 @@ release:ngc-precompiled-ubuntu24.04:
   rules:
   - !reference [.precompiled-rules, rules]
 
+release:ngc-precompiled-ubuntu26.04:
+  variables:
+    DIST: signed_ubuntu26.04
+    BASE_TARGET: questing
+    PRECOMPILED: "true"
+  extends:
+    - .driver-versions-precompiled-ubuntu26.04
+    - .release-generic
+    - .release:ngc-variables
+  rules:
+    - !reference [.precompiled-rules, rules]
+
 release:ngc-precompiled-ubuntu22.04:
   variables:
     DIST: signed_ubuntu22.04