From d1369dfd47667b025052b7267e7649788e31e898 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Thu, 18 Dec 2025 19:52:22 +0000 Subject: [PATCH 01/11] u Signed-off-by: Joe Isaacs --- .github/actions/build-ami/action.yml | 102 ++++++++++++++++++++++++ .github/runs-on.yml | 10 ++- .github/workflows/ami-prebuild.yml | 115 +++++++++++++++++++++++++++ 3 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 .github/actions/build-ami/action.yml create mode 100644 .github/workflows/ami-prebuild.yml diff --git a/.github/actions/build-ami/action.yml b/.github/actions/build-ami/action.yml new file mode 100644 index 00000000000..74d7f1e937c --- /dev/null +++ b/.github/actions/build-ami/action.yml @@ -0,0 +1,102 @@ +name: "Build Vortex CI AMI" +description: "Build a custom Amazon Machine Image for Vortex CI runners" + +inputs: + arch: + description: "Target architecture: x64 or arm64" + required: true + aws-region: + description: "AWS region to build AMI in" + required: false + default: "us-east-1" + ami-prefix: + description: "Prefix for AMI name" + required: false + default: "vortex-ci" + retention-days: + description: "Number of days before AMI is deprecated" + required: false + default: "30" + +outputs: + ami-id: + description: "The ID of the built AMI" + value: ${{ steps.create-ami.outputs.ami_id }} + ami-name: + description: "The name of the built AMI" + value: ${{ steps.create-ami.outputs.ami_name }} + +runs: + using: "composite" + steps: + - name: Setup Rust + uses: ./.github/actions/setup-rust + + - name: Setup flatc + uses: ./.github/actions/setup-flatc + + - name: Install extra dependencies + shell: bash + run: | + sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + cmake \ + ninja-build \ + clang \ + lld \ + llvm + + - name: Install nightly toolchain + shell: bash + run: | + rustup toolchain install nightly + rustup component add --toolchain nightly rustfmt clippy rust-src miri llvm-tools-preview + + - name: Install cargo tools + shell: bash + run: | + cargo install cargo-nextest --locked + cargo install cargo-hack --locked + cargo install grcov --locked + + - name: Get instance ID + id: instance + shell: bash + run: | + INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) + echo "instance_id=$INSTANCE_ID" >> $GITHUB_OUTPUT + + - name: Create AMI + id: create-ami + shell: bash + env: + AWS_REGION: ${{ inputs.aws-region }} + AMI_PREFIX: ${{ inputs.ami-prefix }} + ARCH: ${{ inputs.arch }} + RETENTION_DAYS: ${{ inputs.retention-days }} + run: | + TIMESTAMP=$(date +%Y%m%d-%H%M%S) + AMI_NAME="${AMI_PREFIX}-${ARCH}-${TIMESTAMP}" + DEPRECATION_TIME=$(date -u -d "+${RETENTION_DAYS} days" +%Y-%m-%dT%H:%M:%SZ) + + echo "Creating AMI: $AMI_NAME" + AMI_ID=$(aws ec2 create-image \ + --instance-id "${{ steps.instance.outputs.instance_id }}" \ + --name "$AMI_NAME" \ + --description "Vortex CI runner image for ${ARCH}" \ + --no-reboot \ + --tag-specifications "ResourceType=image,Tags=[{Key=Name,Value=$AMI_NAME},{Key=Environment,Value=ci},{Key=Arch,Value=$ARCH},{Key=ManagedBy,Value=github-actions}]" \ + --query 'ImageId' \ + --output text) + + echo "Waiting for AMI to be available..." + aws ec2 wait image-available --image-ids "$AMI_ID" + + echo "Setting deprecation time to $DEPRECATION_TIME" + aws ec2 enable-image-deprecation \ + --image-id "$AMI_ID" \ + --deprecate-at "$DEPRECATION_TIME" + + echo "ami_id=$AMI_ID" >> $GITHUB_OUTPUT + echo "ami_name=$AMI_NAME" >> $GITHUB_OUTPUT + echo "AMI created: $AMI_ID ($AMI_NAME)" diff --git a/.github/runs-on.yml b/.github/runs-on.yml index 7970bf75e17..bad0ddf2b9f 100644 --- a/.github/runs-on.yml +++ b/.github/runs-on.yml @@ -1,11 +1,17 @@ +# Custom AMIs for Vortex CI runners +# These AMIs are automatically rebuilt every 15 days by the ami-prebuild.yml workflow +# to keep the GitHub Actions runner agent up to date (required to be <30 days old). +# +# AMI naming pattern: vortex-ci-{arch}-{timestamp} +# Built with: .github/actions/build-ami and .github/packer/vortex-ci.pkr.hcl images: vortex-ci-amd64: platform: "linux" arch: "x64" - name: "vortex-ci-*" + name: "vortex-ci-x64-*" owner: "375504701696" vortex-ci-arm64: platform: "linux" arch: "arm64" - name: "vortex-ci-*" + name: "vortex-ci-arm64-*" owner: "375504701696" diff --git a/.github/workflows/ami-prebuild.yml b/.github/workflows/ami-prebuild.yml new file mode 100644 index 00000000000..447e55db19c --- /dev/null +++ b/.github/workflows/ami-prebuild.yml @@ -0,0 +1,115 @@ +name: AMI Prebuild + +# Schedule to run every 15 days to keep runner agent up to date +# GitHub stops routing jobs to runners with agents older than 30 days +on: + # TODO: Remove push trigger after testing + push: + branches: [ji/ami-prebuild] + paths: + - ".github/workflows/ami-prebuild.yml" + - ".github/actions/build-ami/**" + schedule: + # Run at 00:00 UTC on the 1st and 16th of each month (~15 days apart) + - cron: "0 0 1,16 * *" + workflow_dispatch: + inputs: + arch: + description: "Architecture to build (leave empty for both)" + required: false + type: choice + options: + - "" + - x64 + - arm64 + retention-days: + description: "Days until AMI deprecation" + required: false + type: number + default: 30 + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +permissions: + contents: read + id-token: write + +env: + AWS_REGION: us-east-1 + +jobs: + build-x64: + name: "Build AMI (x64)" + if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.arch == '' || github.event.inputs.arch == 'x64' }} + runs-on: + - runs-on=${{ github.run_id }} + - family=m7i+m7i-flex+m7a + - cpu=4 + - image=ubuntu24-full-x64 + - tag=ami-prebuild-x64 + timeout-minutes: 60 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole + aws-region: ${{ env.AWS_REGION }} + + - name: Build AMI + id: build + uses: ./.github/actions/build-ami + with: + arch: x64 + aws-region: ${{ env.AWS_REGION }} + retention-days: ${{ inputs.retention-days || '30' }} + + - name: Summary + run: | + echo "## AMI Build Complete (x64)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- **AMI ID:** ${{ steps.build.outputs.ami-id }}" >> $GITHUB_STEP_SUMMARY + echo "- **AMI Name:** ${{ steps.build.outputs.ami-name }}" >> $GITHUB_STEP_SUMMARY + echo "- **Deprecation:** ${{ inputs.retention-days || '30' }} days" >> $GITHUB_STEP_SUMMARY + + build-arm64: + name: "Build AMI (arm64)" + if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.arch == '' || github.event.inputs.arch == 'arm64' }} + runs-on: + - runs-on=${{ github.run_id }} + - family=m7g + - cpu=4 + - image=ubuntu24-full-arm64 + - tag=ami-prebuild-arm64 + timeout-minutes: 60 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole + aws-region: ${{ env.AWS_REGION }} + + - name: Build AMI + id: build + uses: ./.github/actions/build-ami + with: + arch: arm64 + aws-region: ${{ env.AWS_REGION }} + retention-days: ${{ inputs.retention-days || '30' }} + + - name: Summary + run: | + echo "## AMI Build Complete (arm64)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- **AMI ID:** ${{ steps.build.outputs.ami-id }}" >> $GITHUB_STEP_SUMMARY + echo "- **AMI Name:** ${{ steps.build.outputs.ami-name }}" >> $GITHUB_STEP_SUMMARY + echo "- **Deprecation:** ${{ inputs.retention-days || '30' }} days" >> $GITHUB_STEP_SUMMARY From 3056596b99a75c082cabc86dfa369645f35ab89e Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 19 Dec 2025 12:05:54 +0000 Subject: [PATCH 02/11] Use RUNS_ON_INSTANCE_ID env var instead of metadata service Signed-off-by: Claude --- .github/actions/build-ami/action.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/actions/build-ami/action.yml b/.github/actions/build-ami/action.yml index 74d7f1e937c..427116fa462 100644 --- a/.github/actions/build-ami/action.yml +++ b/.github/actions/build-ami/action.yml @@ -59,13 +59,6 @@ runs: cargo install cargo-hack --locked cargo install grcov --locked - - name: Get instance ID - id: instance - shell: bash - run: | - INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) - echo "instance_id=$INSTANCE_ID" >> $GITHUB_OUTPUT - - name: Create AMI id: create-ami shell: bash @@ -79,9 +72,9 @@ runs: AMI_NAME="${AMI_PREFIX}-${ARCH}-${TIMESTAMP}" DEPRECATION_TIME=$(date -u -d "+${RETENTION_DAYS} days" +%Y-%m-%dT%H:%M:%SZ) - echo "Creating AMI: $AMI_NAME" + echo "Creating AMI: $AMI_NAME from instance $RUNS_ON_INSTANCE_ID" AMI_ID=$(aws ec2 create-image \ - --instance-id "${{ steps.instance.outputs.instance_id }}" \ + --instance-id "$RUNS_ON_INSTANCE_ID" \ --name "$AMI_NAME" \ --description "Vortex CI runner image for ${ARCH}" \ --no-reboot \ From 8c1b88999de715c48ee9d7ce0ce9b31f40aa5a89 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 19 Dec 2025 12:14:30 +0000 Subject: [PATCH 03/11] Use eu-west-1 region and RUNS_ON_AWS_REGION for AMI creation Signed-off-by: Claude --- .github/actions/build-ami/action.yml | 21 ++++++++++++--------- .github/workflows/ami-prebuild.yml | 2 +- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/actions/build-ami/action.yml b/.github/actions/build-ami/action.yml index 427116fa462..7c9fa7ef6fd 100644 --- a/.github/actions/build-ami/action.yml +++ b/.github/actions/build-ami/action.yml @@ -8,7 +8,7 @@ inputs: aws-region: description: "AWS region to build AMI in" required: false - default: "us-east-1" + default: "eu-west-1" ami-prefix: description: "Prefix for AMI name" required: false @@ -52,27 +52,30 @@ runs: rustup toolchain install nightly rustup component add --toolchain nightly rustfmt clippy rust-src miri llvm-tools-preview - - name: Install cargo tools - shell: bash - run: | - cargo install cargo-nextest --locked - cargo install cargo-hack --locked - cargo install grcov --locked +# - name: Install cargo tools +# shell: bash +# run: | +# cargo install cargo-nextest --locked +# cargo install cargo-hack --locked +# cargo install grcov --locked - name: Create AMI id: create-ami shell: bash env: - AWS_REGION: ${{ inputs.aws-region }} AMI_PREFIX: ${{ inputs.ami-prefix }} ARCH: ${{ inputs.arch }} RETENTION_DAYS: ${{ inputs.retention-days }} run: | + # Use the region where the instance is running + export AWS_REGION="$RUNS_ON_AWS_REGION" + export AWS_DEFAULT_REGION="$RUNS_ON_AWS_REGION" + TIMESTAMP=$(date +%Y%m%d-%H%M%S) AMI_NAME="${AMI_PREFIX}-${ARCH}-${TIMESTAMP}" DEPRECATION_TIME=$(date -u -d "+${RETENTION_DAYS} days" +%Y-%m-%dT%H:%M:%SZ) - echo "Creating AMI: $AMI_NAME from instance $RUNS_ON_INSTANCE_ID" + echo "Creating AMI: $AMI_NAME from instance $RUNS_ON_INSTANCE_ID in region $AWS_REGION" AMI_ID=$(aws ec2 create-image \ --instance-id "$RUNS_ON_INSTANCE_ID" \ --name "$AMI_NAME" \ diff --git a/.github/workflows/ami-prebuild.yml b/.github/workflows/ami-prebuild.yml index 447e55db19c..0644ec8031e 100644 --- a/.github/workflows/ami-prebuild.yml +++ b/.github/workflows/ami-prebuild.yml @@ -37,7 +37,7 @@ permissions: id-token: write env: - AWS_REGION: us-east-1 + AWS_REGION: eu-west-1 jobs: build-x64: From 095fcede6d881bb291df496e1935056b61302964 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 19 Dec 2025 12:19:56 +0000 Subject: [PATCH 04/11] Add test jobs to verify AMI works after build Signed-off-by: Claude --- .github/workflows/ami-prebuild.yml | 49 ++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/.github/workflows/ami-prebuild.yml b/.github/workflows/ami-prebuild.yml index 0644ec8031e..19bf36e78fe 100644 --- a/.github/workflows/ami-prebuild.yml +++ b/.github/workflows/ami-prebuild.yml @@ -113,3 +113,52 @@ jobs: echo "- **AMI ID:** ${{ steps.build.outputs.ami-id }}" >> $GITHUB_STEP_SUMMARY echo "- **AMI Name:** ${{ steps.build.outputs.ami-name }}" >> $GITHUB_STEP_SUMMARY echo "- **Deprecation:** ${{ inputs.retention-days || '30' }} days" >> $GITHUB_STEP_SUMMARY + + # Test the newly built AMI by running a simple lint check + test-x64: + name: "Test AMI (x64)" + needs: build-x64 + runs-on: + - runs-on=${{ github.run_id }} + - family=m7i+m7i-flex+m7a + - cpu=4 + - image=vortex-ci-amd64 + - tag=test-ami-x64 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + - name: Verify tools are installed + run: | + echo "Checking installed tools..." + cargo --version + rustc --version + rustup show + protoc --version + flatc --version + mold --version + - name: Run cargo check + run: cargo check --locked + + test-arm64: + name: "Test AMI (arm64)" + needs: build-arm64 + runs-on: + - runs-on=${{ github.run_id }} + - family=m7g + - cpu=4 + - image=vortex-ci-arm64 + - tag=test-ami-arm64 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + - name: Verify tools are installed + run: | + echo "Checking installed tools..." + cargo --version + rustc --version + rustup show + protoc --version + flatc --version + mold --version + - name: Run cargo check + run: cargo check --locked From fbf2a783a4e5ae26c9a267403b52acc69c6e196b Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 19 Dec 2025 12:23:59 +0000 Subject: [PATCH 05/11] u Signed-off-by: Joe Isaacs --- .github/actions/build-ami/action.yml | 12 +++++------ .github/workflows/ami-prebuild.yml | 30 ---------------------------- .github/workflows/ci.yml | 6 +----- 3 files changed, 7 insertions(+), 41 deletions(-) diff --git a/.github/actions/build-ami/action.yml b/.github/actions/build-ami/action.yml index 7c9fa7ef6fd..b70018d211a 100644 --- a/.github/actions/build-ami/action.yml +++ b/.github/actions/build-ami/action.yml @@ -52,12 +52,12 @@ runs: rustup toolchain install nightly rustup component add --toolchain nightly rustfmt clippy rust-src miri llvm-tools-preview -# - name: Install cargo tools -# shell: bash -# run: | -# cargo install cargo-nextest --locked -# cargo install cargo-hack --locked -# cargo install grcov --locked + - name: Install cargo tools + shell: bash + run: | + cargo install cargo-nextest --locked + cargo install cargo-hack --locked + cargo install grcov --locked - name: Create AMI id: create-ami diff --git a/.github/workflows/ami-prebuild.yml b/.github/workflows/ami-prebuild.yml index 19bf36e78fe..93a8ee503f6 100644 --- a/.github/workflows/ami-prebuild.yml +++ b/.github/workflows/ami-prebuild.yml @@ -3,12 +3,6 @@ name: AMI Prebuild # Schedule to run every 15 days to keep runner agent up to date # GitHub stops routing jobs to runners with agents older than 30 days on: - # TODO: Remove push trigger after testing - push: - branches: [ji/ami-prebuild] - paths: - - ".github/workflows/ami-prebuild.yml" - - ".github/actions/build-ami/**" schedule: # Run at 00:00 UTC on the 1st and 16th of each month (~15 days apart) - cron: "0 0 1,16 * *" @@ -115,30 +109,6 @@ jobs: echo "- **Deprecation:** ${{ inputs.retention-days || '30' }} days" >> $GITHUB_STEP_SUMMARY # Test the newly built AMI by running a simple lint check - test-x64: - name: "Test AMI (x64)" - needs: build-x64 - runs-on: - - runs-on=${{ github.run_id }} - - family=m7i+m7i-flex+m7a - - cpu=4 - - image=vortex-ci-amd64 - - tag=test-ami-x64 - timeout-minutes: 30 - steps: - - uses: actions/checkout@v6 - - name: Verify tools are installed - run: | - echo "Checking installed tools..." - cargo --version - rustc --version - rustup show - protoc --version - flatc --version - mold --version - - name: Run cargo check - run: cargo check --locked - test-arm64: name: "Test AMI (arm64)" needs: build-arm64 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ad48e0efd5..8347ec61dc3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -265,7 +265,7 @@ jobs: - runs-on=${{ github.run_id }} - family=m7i+m7i-flex+m7a - cpu=16 - - image=ubuntu24-full-x64 + - image=vortex-ci-amd64 - extras=s3-cache - tag=rust-lint steps: @@ -273,10 +273,6 @@ jobs: with: sccache: s3 - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-rust - with: - toolchain: nightly - repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Rust Lint - Format run: cargo +nightly fmt --all --check - name: Rustc check From 5ac2076cdaf8c6a1d6e111545dfa57b368d0f42e Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 19 Dec 2025 12:37:38 +0000 Subject: [PATCH 06/11] u Signed-off-by: Joe Isaacs --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8347ec61dc3..9dd9385d811 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -191,7 +191,7 @@ jobs: - runs-on=${{ github.run_id }} - family=m7i+m7i-flex+m7a - cpu=8 - - image=ubuntu24-full-x64 + - image=vortex-ci-amd64 - extras=s3-cache - tag=${{ matrix.config.name }} env: @@ -291,7 +291,7 @@ jobs: - runs-on=${{ github.run_id }} - family=m7i+m7i-flex+m7a - cpu=16 - - image=ubuntu24-full-x64 + - image=vortex-ci-amd64 - extras=s3-cache - tag=rust-lint-no-default steps: From ecdd7c093857be6d95140fa6cf0ba90b05500b99 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 19 Dec 2025 12:50:53 +0000 Subject: [PATCH 07/11] Clean runner state before AMI creation and add push trigger - Stop runner service and remove registration/credentials - Clear temp files and bash history - Add push trigger for testing on ji/ami-prebuild branch Signed-off-by: Claude --- .github/actions/build-ami/action.yml | 22 ++++++++++++++++++++++ .github/workflows/ami-prebuild.yml | 6 ++++++ 2 files changed, 28 insertions(+) diff --git a/.github/actions/build-ami/action.yml b/.github/actions/build-ami/action.yml index b70018d211a..f0f1503627d 100644 --- a/.github/actions/build-ami/action.yml +++ b/.github/actions/build-ami/action.yml @@ -59,6 +59,28 @@ runs: cargo install cargo-hack --locked cargo install grcov --locked + - name: Clean runner state before AMI creation + shell: bash + run: | + # Stop the GitHub Actions runner service + sudo systemctl stop actions.runner.* || true + + # Remove runner registration and credentials + sudo rm -rf /home/runner/actions-runner/.runner || true + sudo rm -rf /home/runner/actions-runner/.credentials* || true + sudo rm -rf /home/runner/actions-runner/_diag || true + sudo rm -rf /home/runner/actions-runner/_work || true + + # Clear any cached runner state + sudo rm -rf /tmp/* || true + sudo rm -rf /var/tmp/* || true + + # Clear bash history + cat /dev/null > ~/.bash_history || true + history -c || true + + echo "Runner state cleaned for AMI creation" + - name: Create AMI id: create-ami shell: bash diff --git a/.github/workflows/ami-prebuild.yml b/.github/workflows/ami-prebuild.yml index 93a8ee503f6..bc7e403da0b 100644 --- a/.github/workflows/ami-prebuild.yml +++ b/.github/workflows/ami-prebuild.yml @@ -3,6 +3,12 @@ name: AMI Prebuild # Schedule to run every 15 days to keep runner agent up to date # GitHub stops routing jobs to runners with agents older than 30 days on: + # TODO: Remove push trigger after testing + push: + branches: [ji/ami-prebuild] + paths: + - ".github/workflows/ami-prebuild.yml" + - ".github/actions/build-ami/**" schedule: # Run at 00:00 UTC on the 1st and 16th of each month (~15 days apart) - cron: "0 0 1,16 * *" From 54c4b07a3b21f453372fbab762fe091031ddac7d Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 19 Dec 2025 13:04:45 +0000 Subject: [PATCH 08/11] u Signed-off-by: Joe Isaacs --- .github/actions/build-ami/action.yml | 120 ----------------------- .github/packer/scripts/provision.sh | 84 +++++++++++++++++ .github/packer/scripts/user_data.sh | 3 + .github/packer/vortex-ci.pkr.hcl | 120 +++++++++++++++++++++++ .github/workflows/ami-prebuild.yml | 136 +++++++++++++++++++++------ .github/workflows/ci.yml | 4 +- 6 files changed, 314 insertions(+), 153 deletions(-) delete mode 100644 .github/actions/build-ami/action.yml create mode 100755 .github/packer/scripts/provision.sh create mode 100755 .github/packer/scripts/user_data.sh create mode 100644 .github/packer/vortex-ci.pkr.hcl diff --git a/.github/actions/build-ami/action.yml b/.github/actions/build-ami/action.yml deleted file mode 100644 index f0f1503627d..00000000000 --- a/.github/actions/build-ami/action.yml +++ /dev/null @@ -1,120 +0,0 @@ -name: "Build Vortex CI AMI" -description: "Build a custom Amazon Machine Image for Vortex CI runners" - -inputs: - arch: - description: "Target architecture: x64 or arm64" - required: true - aws-region: - description: "AWS region to build AMI in" - required: false - default: "eu-west-1" - ami-prefix: - description: "Prefix for AMI name" - required: false - default: "vortex-ci" - retention-days: - description: "Number of days before AMI is deprecated" - required: false - default: "30" - -outputs: - ami-id: - description: "The ID of the built AMI" - value: ${{ steps.create-ami.outputs.ami_id }} - ami-name: - description: "The name of the built AMI" - value: ${{ steps.create-ami.outputs.ami_name }} - -runs: - using: "composite" - steps: - - name: Setup Rust - uses: ./.github/actions/setup-rust - - - name: Setup flatc - uses: ./.github/actions/setup-flatc - - - name: Install extra dependencies - shell: bash - run: | - sudo apt-get update - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - cmake \ - ninja-build \ - clang \ - lld \ - llvm - - - name: Install nightly toolchain - shell: bash - run: | - rustup toolchain install nightly - rustup component add --toolchain nightly rustfmt clippy rust-src miri llvm-tools-preview - - - name: Install cargo tools - shell: bash - run: | - cargo install cargo-nextest --locked - cargo install cargo-hack --locked - cargo install grcov --locked - - - name: Clean runner state before AMI creation - shell: bash - run: | - # Stop the GitHub Actions runner service - sudo systemctl stop actions.runner.* || true - - # Remove runner registration and credentials - sudo rm -rf /home/runner/actions-runner/.runner || true - sudo rm -rf /home/runner/actions-runner/.credentials* || true - sudo rm -rf /home/runner/actions-runner/_diag || true - sudo rm -rf /home/runner/actions-runner/_work || true - - # Clear any cached runner state - sudo rm -rf /tmp/* || true - sudo rm -rf /var/tmp/* || true - - # Clear bash history - cat /dev/null > ~/.bash_history || true - history -c || true - - echo "Runner state cleaned for AMI creation" - - - name: Create AMI - id: create-ami - shell: bash - env: - AMI_PREFIX: ${{ inputs.ami-prefix }} - ARCH: ${{ inputs.arch }} - RETENTION_DAYS: ${{ inputs.retention-days }} - run: | - # Use the region where the instance is running - export AWS_REGION="$RUNS_ON_AWS_REGION" - export AWS_DEFAULT_REGION="$RUNS_ON_AWS_REGION" - - TIMESTAMP=$(date +%Y%m%d-%H%M%S) - AMI_NAME="${AMI_PREFIX}-${ARCH}-${TIMESTAMP}" - DEPRECATION_TIME=$(date -u -d "+${RETENTION_DAYS} days" +%Y-%m-%dT%H:%M:%SZ) - - echo "Creating AMI: $AMI_NAME from instance $RUNS_ON_INSTANCE_ID in region $AWS_REGION" - AMI_ID=$(aws ec2 create-image \ - --instance-id "$RUNS_ON_INSTANCE_ID" \ - --name "$AMI_NAME" \ - --description "Vortex CI runner image for ${ARCH}" \ - --no-reboot \ - --tag-specifications "ResourceType=image,Tags=[{Key=Name,Value=$AMI_NAME},{Key=Environment,Value=ci},{Key=Arch,Value=$ARCH},{Key=ManagedBy,Value=github-actions}]" \ - --query 'ImageId' \ - --output text) - - echo "Waiting for AMI to be available..." - aws ec2 wait image-available --image-ids "$AMI_ID" - - echo "Setting deprecation time to $DEPRECATION_TIME" - aws ec2 enable-image-deprecation \ - --image-id "$AMI_ID" \ - --deprecate-at "$DEPRECATION_TIME" - - echo "ami_id=$AMI_ID" >> $GITHUB_OUTPUT - echo "ami_name=$AMI_NAME" >> $GITHUB_OUTPUT - echo "AMI created: $AMI_ID ($AMI_NAME)" diff --git a/.github/packer/scripts/provision.sh b/.github/packer/scripts/provision.sh new file mode 100755 index 00000000000..682917b6170 --- /dev/null +++ b/.github/packer/scripts/provision.sh @@ -0,0 +1,84 @@ +#!/bin/bash +set -euo pipefail + +# Variables (passed from Packer) +RUST_TOOLCHAIN="${RUST_TOOLCHAIN:-1.89}" +PROTOC_VERSION="${PROTOC_VERSION:-29.3}" +FLATC_VERSION="${FLATC_VERSION:-25.9.23}" + +echo "=== Installing Vortex CI dependencies ===" + +# Install build dependencies +echo "Installing system packages..." +sudo apt-get update +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + cmake \ + ninja-build \ + clang \ + lld \ + llvm \ + pkg-config \ + libssl-dev + +# Install Rust +echo "Installing Rust ${RUST_TOOLCHAIN}..." +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain "${RUST_TOOLCHAIN}" +source "$HOME/.cargo/env" +echo 'source $HOME/.cargo/env' >> "$HOME/.bashrc" + +# Install Rust components +rustup component add clippy rustfmt +rustup toolchain install nightly +rustup component add --toolchain nightly rustfmt clippy rust-src miri llvm-tools-preview + +echo "Rust installed:" +cargo --version +rustc --version + +# Install protoc +echo "Installing protoc ${PROTOC_VERSION}..." +ARCH=$(uname -m) +if [ "$ARCH" = "x86_64" ]; then + PROTOC_ARCH=linux-x86_64 +else + PROTOC_ARCH=linux-aarch_64 +fi +curl -fsSL -o /tmp/protoc.zip "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${PROTOC_ARCH}.zip" +sudo unzip -o /tmp/protoc.zip -d /usr/local bin/protoc 'include/*' +sudo chmod +x /usr/local/bin/protoc +rm /tmp/protoc.zip +protoc --version + +# Install flatc +echo "Installing flatc ${FLATC_VERSION}..." +if [ "$ARCH" = "x86_64" ]; then + curl -fsSL -o /tmp/flatc.zip "https://github.com/google/flatbuffers/releases/download/v${FLATC_VERSION}/Linux.flatc.binary.clang++-18.zip" + sudo unzip -o /tmp/flatc.zip -d /usr/local/bin + sudo chmod +x /usr/local/bin/flatc + rm /tmp/flatc.zip +else + # Build from source for ARM64 + git clone --depth 1 --branch "v${FLATC_VERSION}" https://github.com/google/flatbuffers.git /tmp/flatbuffers + cd /tmp/flatbuffers + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release . + ninja + sudo cp flatc /usr/local/bin/ + cd - + rm -rf /tmp/flatbuffers +fi +flatc --version + +# Install cargo tools +echo "Installing cargo tools..." +source "$HOME/.cargo/env" +cargo install cargo-nextest --locked +cargo install cargo-hack --locked +cargo install grcov --locked + +# Cleanup +echo "Cleaning up..." +sudo apt-get clean +sudo rm -rf /var/lib/apt/lists/* +rm -rf /tmp/* + +echo "=== Vortex CI dependencies installed successfully ===" diff --git a/.github/packer/scripts/user_data.sh b/.github/packer/scripts/user_data.sh new file mode 100755 index 00000000000..03e4aa50f7e --- /dev/null +++ b/.github/packer/scripts/user_data.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# Start SSH for Packer to connect +systemctl start ssh diff --git a/.github/packer/vortex-ci.pkr.hcl b/.github/packer/vortex-ci.pkr.hcl new file mode 100644 index 00000000000..dbfaf37c7c3 --- /dev/null +++ b/.github/packer/vortex-ci.pkr.hcl @@ -0,0 +1,120 @@ +packer { + required_plugins { + amazon = { + version = ">= 1.3.0" + source = "github.com/hashicorp/amazon" + } + } +} + +variable "aws_region" { + type = string + default = "eu-west-1" +} + +variable "arch" { + type = string + description = "Architecture: x64 or arm64" +} + +variable "ami_prefix" { + type = string + default = "vortex-ci" +} + +variable "source_ami_owner" { + type = string + default = "135269210855" + description = "runs-on AWS account ID" +} + +variable "subnet_id" { + type = string + default = "" +} + +variable "rust_toolchain" { + type = string + default = "1.89" +} + +variable "protoc_version" { + type = string + default = "29.3" +} + +variable "flatc_version" { + type = string + default = "25.9.23" +} + +locals { + timestamp = formatdate("YYYYMMDD-HHmmss", timestamp()) + + arch_config = { + x64 = { + instance_type = "m7i.large" + source_ami_name = "runs-on-v2.2-ubuntu24-full-x64-*" + ami_arch = "x86_64" + } + arm64 = { + instance_type = "m7g.large" + source_ami_name = "runs-on-v2.2-ubuntu24-full-arm64-*" + ami_arch = "arm64" + } + } + + config = local.arch_config[var.arch] +} + +source "amazon-ebs" "vortex-ci" { + ami_name = "${var.ami_prefix}-${var.arch}-${local.timestamp}" + instance_type = local.config.instance_type + region = var.aws_region + + source_ami_filter { + filters = { + name = local.config.source_ami_name + root-device-type = "ebs" + virtualization-type = "hvm" + architecture = local.config.ami_arch + } + most_recent = true + owners = [var.source_ami_owner] + } + + subnet_id = var.subnet_id != "" ? var.subnet_id : null + ssh_username = "runner" + + # User data to start SSH for Packer connectivity + user_data_file = "${path.root}/scripts/user_data.sh" + + launch_block_device_mappings { + device_name = "/dev/sda1" + volume_size = 80 + volume_type = "gp3" + delete_on_termination = true + } + + tags = { + Name = "${var.ami_prefix}-${var.arch}" + Environment = "ci" + Arch = var.arch + BuildTime = local.timestamp + ManagedBy = "packer" + } +} + +build { + sources = ["source.amazon-ebs.vortex-ci"] + + # Run the provisioning script + provisioner "shell" { + script = "${path.root}/scripts/provision.sh" + environment_vars = [ + "RUST_TOOLCHAIN=${var.rust_toolchain}", + "PROTOC_VERSION=${var.protoc_version}", + "FLATC_VERSION=${var.flatc_version}" + ] + } +} diff --git a/.github/workflows/ami-prebuild.yml b/.github/workflows/ami-prebuild.yml index bc7e403da0b..e77fdd3160a 100644 --- a/.github/workflows/ami-prebuild.yml +++ b/.github/workflows/ami-prebuild.yml @@ -8,7 +8,7 @@ on: branches: [ji/ami-prebuild] paths: - ".github/workflows/ami-prebuild.yml" - - ".github/actions/build-ami/**" + - ".github/packer/**" schedule: # Run at 00:00 UTC on the 1st and 16th of each month (~15 days apart) - cron: "0 0 1,16 * *" @@ -43,13 +43,11 @@ jobs: build-x64: name: "Build AMI (x64)" if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.arch == '' || github.event.inputs.arch == 'x64' }} - runs-on: - - runs-on=${{ github.run_id }} - - family=m7i+m7i-flex+m7a - - cpu=4 - - image=ubuntu24-full-x64 - - tag=ami-prebuild-x64 + runs-on: ubuntu-latest timeout-minutes: 60 + outputs: + ami-id: ${{ steps.build.outputs.ami_id }} + ami-name: ${{ steps.build.outputs.ami_name }} steps: - name: Checkout @@ -61,32 +59,58 @@ jobs: role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole aws-region: ${{ env.AWS_REGION }} - - name: Build AMI - id: build - uses: ./.github/actions/build-ami + - name: Setup Packer + uses: hashicorp/setup-packer@main with: - arch: x64 - aws-region: ${{ env.AWS_REGION }} - retention-days: ${{ inputs.retention-days || '30' }} + version: "1.11.2" + + - name: Packer Init + working-directory: .github/packer + run: packer init vortex-ci.pkr.hcl + + - name: Packer Build + id: build + working-directory: .github/packer + run: | + packer build \ + -var "arch=x64" \ + -var "aws_region=${{ env.AWS_REGION }}" \ + -var "subnet_id=${{ secrets.AWS_SUBNET_ID }}" \ + -machine-readable \ + vortex-ci.pkr.hcl | tee packer-output.log + + # Extract AMI ID from Packer output + AMI_ID=$(grep 'artifact,0,id' packer-output.log | tail -1 | cut -d',' -f6 | cut -d':' -f2) + AMI_NAME=$(grep 'artifact,0,string' packer-output.log | tail -1 | grep -oP 'AMIs were created:.*' | sed 's/AMIs were created:\\n\\n//' | head -1 || echo "vortex-ci-x64") + + echo "ami_id=$AMI_ID" >> $GITHUB_OUTPUT + echo "ami_name=$AMI_NAME" >> $GITHUB_OUTPUT + echo "Built AMI: $AMI_ID" + + - name: Set AMI Deprecation + run: | + RETENTION_DAYS=${{ inputs.retention-days || '30' }} + DEPRECATION_TIME=$(date -u -d "+${RETENTION_DAYS} days" +%Y-%m-%dT%H:%M:%SZ) + aws ec2 enable-image-deprecation \ + --image-id "${{ steps.build.outputs.ami_id }}" \ + --deprecate-at "$DEPRECATION_TIME" + echo "AMI will be deprecated at $DEPRECATION_TIME" - name: Summary run: | echo "## AMI Build Complete (x64)" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "- **AMI ID:** ${{ steps.build.outputs.ami-id }}" >> $GITHUB_STEP_SUMMARY - echo "- **AMI Name:** ${{ steps.build.outputs.ami-name }}" >> $GITHUB_STEP_SUMMARY + echo "- **AMI ID:** ${{ steps.build.outputs.ami_id }}" >> $GITHUB_STEP_SUMMARY echo "- **Deprecation:** ${{ inputs.retention-days || '30' }} days" >> $GITHUB_STEP_SUMMARY build-arm64: name: "Build AMI (arm64)" if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.arch == '' || github.event.inputs.arch == 'arm64' }} - runs-on: - - runs-on=${{ github.run_id }} - - family=m7g - - cpu=4 - - image=ubuntu24-full-arm64 - - tag=ami-prebuild-arm64 + runs-on: ubuntu-latest timeout-minutes: 60 + outputs: + ami-id: ${{ steps.build.outputs.ami_id }} + ami-name: ${{ steps.build.outputs.ami_name }} steps: - name: Checkout @@ -98,23 +122,74 @@ jobs: role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole aws-region: ${{ env.AWS_REGION }} - - name: Build AMI - id: build - uses: ./.github/actions/build-ami + - name: Setup Packer + uses: hashicorp/setup-packer@main with: - arch: arm64 - aws-region: ${{ env.AWS_REGION }} - retention-days: ${{ inputs.retention-days || '30' }} + version: "1.11.2" + + - name: Packer Init + working-directory: .github/packer + run: packer init vortex-ci.pkr.hcl + + - name: Packer Build + id: build + working-directory: .github/packer + run: | + packer build \ + -var "arch=arm64" \ + -var "aws_region=${{ env.AWS_REGION }}" \ + -var "subnet_id=${{ secrets.AWS_SUBNET_ID }}" \ + -machine-readable \ + vortex-ci.pkr.hcl | tee packer-output.log + + # Extract AMI ID from Packer output + AMI_ID=$(grep 'artifact,0,id' packer-output.log | tail -1 | cut -d',' -f6 | cut -d':' -f2) + AMI_NAME=$(grep 'artifact,0,string' packer-output.log | tail -1 | grep -oP 'AMIs were created:.*' | sed 's/AMIs were created:\\n\\n//' | head -1 || echo "vortex-ci-arm64") + + echo "ami_id=$AMI_ID" >> $GITHUB_OUTPUT + echo "ami_name=$AMI_NAME" >> $GITHUB_OUTPUT + echo "Built AMI: $AMI_ID" + + - name: Set AMI Deprecation + run: | + RETENTION_DAYS=${{ inputs.retention-days || '30' }} + DEPRECATION_TIME=$(date -u -d "+${RETENTION_DAYS} days" +%Y-%m-%dT%H:%M:%SZ) + aws ec2 enable-image-deprecation \ + --image-id "${{ steps.build.outputs.ami_id }}" \ + --deprecate-at "$DEPRECATION_TIME" + echo "AMI will be deprecated at $DEPRECATION_TIME" - name: Summary run: | echo "## AMI Build Complete (arm64)" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "- **AMI ID:** ${{ steps.build.outputs.ami-id }}" >> $GITHUB_STEP_SUMMARY - echo "- **AMI Name:** ${{ steps.build.outputs.ami-name }}" >> $GITHUB_STEP_SUMMARY + echo "- **AMI ID:** ${{ steps.build.outputs.ami_id }}" >> $GITHUB_STEP_SUMMARY echo "- **Deprecation:** ${{ inputs.retention-days || '30' }} days" >> $GITHUB_STEP_SUMMARY - # Test the newly built AMI by running a simple lint check + # Test the newly built AMI + test-x64: + name: "Test AMI (x64)" + needs: build-x64 + runs-on: + - runs-on=${{ github.run_id }} + - family=m7i+m7i-flex+m7a + - cpu=4 + - image=vortex-ci-amd64 + - tag=test-ami-x64 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + - name: Verify tools are installed + run: | + echo "Checking installed tools..." + cargo --version + rustc --version + rustup show + protoc --version + flatc --version + - name: Run cargo check + run: cargo check --locked + test-arm64: name: "Test AMI (arm64)" needs: build-arm64 @@ -135,6 +210,5 @@ jobs: rustup show protoc --version flatc --version - mold --version - name: Run cargo check run: cargo check --locked diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9dd9385d811..45203f89f85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -191,7 +191,7 @@ jobs: - runs-on=${{ github.run_id }} - family=m7i+m7i-flex+m7a - cpu=8 - - image=vortex-ci-amd64 + - image=ubuntu24-full-x64 - extras=s3-cache - tag=${{ matrix.config.name }} env: @@ -265,7 +265,7 @@ jobs: - runs-on=${{ github.run_id }} - family=m7i+m7i-flex+m7a - cpu=16 - - image=vortex-ci-amd64 + - image=ubuntu24-full-x64 - extras=s3-cache - tag=rust-lint steps: From 8431dfbd579a359d21f5c4874cbb8cec62078a7f Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 19 Dec 2025 13:23:27 +0000 Subject: [PATCH 09/11] Switch AMI build back to simple approach with runner cleanup - Revert from Packer to simple create-image approach - Build jobs now run on runs-on instances directly - Added runner state cleanup before AMI creation to fix registration issues - Removes .runner, .credentials files so new instances register fresh Signed-off-by: Joe Isaacs --- .github/actions/build-ami/action.yml | 128 +++++++++++++++++++++++++++ .github/workflows/ami-prebuild.yml | 109 +++++++---------------- 2 files changed, 159 insertions(+), 78 deletions(-) create mode 100644 .github/actions/build-ami/action.yml diff --git a/.github/actions/build-ami/action.yml b/.github/actions/build-ami/action.yml new file mode 100644 index 00000000000..96a529ee5a2 --- /dev/null +++ b/.github/actions/build-ami/action.yml @@ -0,0 +1,128 @@ +name: "Build Vortex CI AMI" +description: "Build a custom Amazon Machine Image for Vortex CI runners" + +inputs: + arch: + description: "Target architecture: x64 or arm64" + required: true + ami-prefix: + description: "Prefix for AMI name" + required: false + default: "vortex-ci" + retention-days: + description: "Number of days before AMI is deprecated" + required: false + default: "30" + +outputs: + ami-id: + description: "The ID of the built AMI" + value: ${{ steps.create-ami.outputs.ami_id }} + ami-name: + description: "The name of the built AMI" + value: ${{ steps.create-ami.outputs.ami_name }} + +runs: + using: "composite" + steps: + - name: Setup Rust + uses: ./.github/actions/setup-rust + + - name: Setup flatc + uses: ./.github/actions/setup-flatc + + - name: Install extra dependencies + shell: bash + run: | + sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + cmake \ + ninja-build \ + clang \ + lld \ + llvm + + - name: Install nightly toolchain + shell: bash + run: | + rustup toolchain install nightly + rustup component add --toolchain nightly rustfmt clippy rust-src miri llvm-tools-preview + + - name: Install cargo tools + shell: bash + run: | + cargo install cargo-nextest --locked + cargo install cargo-hack --locked + cargo install grcov --locked + + - name: Clean runner for AMI + shell: bash + run: | + echo "=== Cleaning runner state for clean AMI ===" + + # The runner service name + RUNNER_SVC=$(systemctl list-units --type=service | grep actions.runner | awk '{print $1}' | head -1) + + if [ -n "$RUNNER_SVC" ]; then + echo "Stopping runner service: $RUNNER_SVC" + sudo systemctl stop "$RUNNER_SVC" || true + fi + + # Remove runner registration (this is the key!) + RUNNER_DIR="/home/runner/actions-runner" + if [ -d "$RUNNER_DIR" ]; then + echo "Cleaning runner directory..." + sudo rm -f "$RUNNER_DIR/.runner" || true + sudo rm -f "$RUNNER_DIR/.credentials" || true + sudo rm -f "$RUNNER_DIR/.credentials_rsaparams" || true + sudo rm -rf "$RUNNER_DIR/_diag" || true + sudo rm -rf "$RUNNER_DIR/_work" || true + fi + + # Clear temp files + sudo rm -rf /tmp/* || true + sudo rm -rf /var/tmp/* || true + + # Clear apt cache to reduce AMI size + sudo apt-get clean + sudo rm -rf /var/lib/apt/lists/* + + echo "=== Runner state cleaned ===" + + - name: Create AMI + id: create-ami + shell: bash + env: + AMI_PREFIX: ${{ inputs.ami-prefix }} + ARCH: ${{ inputs.arch }} + RETENTION_DAYS: ${{ inputs.retention-days }} + run: | + export AWS_REGION="$RUNS_ON_AWS_REGION" + export AWS_DEFAULT_REGION="$RUNS_ON_AWS_REGION" + + TIMESTAMP=$(date +%Y%m%d-%H%M%S) + AMI_NAME="${AMI_PREFIX}-${ARCH}-${TIMESTAMP}" + DEPRECATION_TIME=$(date -u -d "+${RETENTION_DAYS} days" +%Y-%m-%dT%H:%M:%SZ) + + echo "Creating AMI: $AMI_NAME from instance $RUNS_ON_INSTANCE_ID in region $AWS_REGION" + + AMI_ID=$(aws ec2 create-image \ + --instance-id "$RUNS_ON_INSTANCE_ID" \ + --name "$AMI_NAME" \ + --description "Vortex CI runner image for ${ARCH}" \ + --no-reboot \ + --tag-specifications "ResourceType=image,Tags=[{Key=Name,Value=$AMI_NAME},{Key=Environment,Value=ci},{Key=Arch,Value=$ARCH},{Key=ManagedBy,Value=github-actions}]" \ + --query 'ImageId' \ + --output text) + + echo "Waiting for AMI to be available..." + aws ec2 wait image-available --image-ids "$AMI_ID" + + echo "Setting deprecation time to $DEPRECATION_TIME" + aws ec2 enable-image-deprecation \ + --image-id "$AMI_ID" \ + --deprecate-at "$DEPRECATION_TIME" + + echo "ami_id=$AMI_ID" >> $GITHUB_OUTPUT + echo "ami_name=$AMI_NAME" >> $GITHUB_OUTPUT + echo "AMI created: $AMI_ID ($AMI_NAME)" diff --git a/.github/workflows/ami-prebuild.yml b/.github/workflows/ami-prebuild.yml index e77fdd3160a..7abd416310a 100644 --- a/.github/workflows/ami-prebuild.yml +++ b/.github/workflows/ami-prebuild.yml @@ -40,14 +40,19 @@ env: AWS_REGION: eu-west-1 jobs: + # Build AMI by running on a runs-on instance and creating an image from it build-x64: name: "Build AMI (x64)" if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.arch == '' || github.event.inputs.arch == 'x64' }} - runs-on: ubuntu-latest + runs-on: + - runs-on=${{ github.run_id }} + - runner=2cpu-linux-x64 + - family=m7i+m7i-flex+m7a + - tag=ami-build-x64 timeout-minutes: 60 outputs: - ami-id: ${{ steps.build.outputs.ami_id }} - ami-name: ${{ steps.build.outputs.ami_name }} + ami-id: ${{ steps.build-ami.outputs.ami-id }} + ami-name: ${{ steps.build-ami.outputs.ami-name }} steps: - name: Checkout @@ -59,58 +64,34 @@ jobs: role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole aws-region: ${{ env.AWS_REGION }} - - name: Setup Packer - uses: hashicorp/setup-packer@main + - name: Build AMI + id: build-ami + uses: ./.github/actions/build-ami with: - version: "1.11.2" - - - name: Packer Init - working-directory: .github/packer - run: packer init vortex-ci.pkr.hcl - - - name: Packer Build - id: build - working-directory: .github/packer - run: | - packer build \ - -var "arch=x64" \ - -var "aws_region=${{ env.AWS_REGION }}" \ - -var "subnet_id=${{ secrets.AWS_SUBNET_ID }}" \ - -machine-readable \ - vortex-ci.pkr.hcl | tee packer-output.log - - # Extract AMI ID from Packer output - AMI_ID=$(grep 'artifact,0,id' packer-output.log | tail -1 | cut -d',' -f6 | cut -d':' -f2) - AMI_NAME=$(grep 'artifact,0,string' packer-output.log | tail -1 | grep -oP 'AMIs were created:.*' | sed 's/AMIs were created:\\n\\n//' | head -1 || echo "vortex-ci-x64") - - echo "ami_id=$AMI_ID" >> $GITHUB_OUTPUT - echo "ami_name=$AMI_NAME" >> $GITHUB_OUTPUT - echo "Built AMI: $AMI_ID" - - - name: Set AMI Deprecation - run: | - RETENTION_DAYS=${{ inputs.retention-days || '30' }} - DEPRECATION_TIME=$(date -u -d "+${RETENTION_DAYS} days" +%Y-%m-%dT%H:%M:%SZ) - aws ec2 enable-image-deprecation \ - --image-id "${{ steps.build.outputs.ami_id }}" \ - --deprecate-at "$DEPRECATION_TIME" - echo "AMI will be deprecated at $DEPRECATION_TIME" + arch: x64 + ami-prefix: vortex-ci + retention-days: ${{ inputs.retention-days || '30' }} - name: Summary run: | echo "## AMI Build Complete (x64)" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "- **AMI ID:** ${{ steps.build.outputs.ami_id }}" >> $GITHUB_STEP_SUMMARY + echo "- **AMI ID:** ${{ steps.build-ami.outputs.ami-id }}" >> $GITHUB_STEP_SUMMARY + echo "- **AMI Name:** ${{ steps.build-ami.outputs.ami-name }}" >> $GITHUB_STEP_SUMMARY echo "- **Deprecation:** ${{ inputs.retention-days || '30' }} days" >> $GITHUB_STEP_SUMMARY build-arm64: name: "Build AMI (arm64)" if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.arch == '' || github.event.inputs.arch == 'arm64' }} - runs-on: ubuntu-latest + runs-on: + - runs-on=${{ github.run_id }} + - runner=2cpu-linux-arm64 + - family=m7g + - tag=ami-build-arm64 timeout-minutes: 60 outputs: - ami-id: ${{ steps.build.outputs.ami_id }} - ami-name: ${{ steps.build.outputs.ami_name }} + ami-id: ${{ steps.build-ami.outputs.ami-id }} + ami-name: ${{ steps.build-ami.outputs.ami-name }} steps: - name: Checkout @@ -122,48 +103,20 @@ jobs: role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole aws-region: ${{ env.AWS_REGION }} - - name: Setup Packer - uses: hashicorp/setup-packer@main + - name: Build AMI + id: build-ami + uses: ./.github/actions/build-ami with: - version: "1.11.2" - - - name: Packer Init - working-directory: .github/packer - run: packer init vortex-ci.pkr.hcl - - - name: Packer Build - id: build - working-directory: .github/packer - run: | - packer build \ - -var "arch=arm64" \ - -var "aws_region=${{ env.AWS_REGION }}" \ - -var "subnet_id=${{ secrets.AWS_SUBNET_ID }}" \ - -machine-readable \ - vortex-ci.pkr.hcl | tee packer-output.log - - # Extract AMI ID from Packer output - AMI_ID=$(grep 'artifact,0,id' packer-output.log | tail -1 | cut -d',' -f6 | cut -d':' -f2) - AMI_NAME=$(grep 'artifact,0,string' packer-output.log | tail -1 | grep -oP 'AMIs were created:.*' | sed 's/AMIs were created:\\n\\n//' | head -1 || echo "vortex-ci-arm64") - - echo "ami_id=$AMI_ID" >> $GITHUB_OUTPUT - echo "ami_name=$AMI_NAME" >> $GITHUB_OUTPUT - echo "Built AMI: $AMI_ID" - - - name: Set AMI Deprecation - run: | - RETENTION_DAYS=${{ inputs.retention-days || '30' }} - DEPRECATION_TIME=$(date -u -d "+${RETENTION_DAYS} days" +%Y-%m-%dT%H:%M:%SZ) - aws ec2 enable-image-deprecation \ - --image-id "${{ steps.build.outputs.ami_id }}" \ - --deprecate-at "$DEPRECATION_TIME" - echo "AMI will be deprecated at $DEPRECATION_TIME" + arch: arm64 + ami-prefix: vortex-ci + retention-days: ${{ inputs.retention-days || '30' }} - name: Summary run: | echo "## AMI Build Complete (arm64)" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "- **AMI ID:** ${{ steps.build.outputs.ami_id }}" >> $GITHUB_STEP_SUMMARY + echo "- **AMI ID:** ${{ steps.build-ami.outputs.ami-id }}" >> $GITHUB_STEP_SUMMARY + echo "- **AMI Name:** ${{ steps.build-ami.outputs.ami-name }}" >> $GITHUB_STEP_SUMMARY echo "- **Deprecation:** ${{ inputs.retention-days || '30' }} days" >> $GITHUB_STEP_SUMMARY # Test the newly built AMI From 78d342033da19bd7b3dcc4e18b30ad2b1ec0193c Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 19 Dec 2025 13:36:11 +0000 Subject: [PATCH 10/11] Switch back to Packer for AMI builds - Use Packer with existing security group to avoid CreateSecurityGroup permission - Pass security_group_id via secret AWS_SECURITY_GROUP_ID - Remove unused build-ami action Signed-off-by: Joe Isaacs --- .github/actions/build-ami/action.yml | 128 --------------------------- .github/packer/vortex-ci.pkr.hcl | 11 ++- .github/workflows/ami-prebuild.yml | 107 +++++++++++++++------- 3 files changed, 85 insertions(+), 161 deletions(-) delete mode 100644 .github/actions/build-ami/action.yml diff --git a/.github/actions/build-ami/action.yml b/.github/actions/build-ami/action.yml deleted file mode 100644 index 96a529ee5a2..00000000000 --- a/.github/actions/build-ami/action.yml +++ /dev/null @@ -1,128 +0,0 @@ -name: "Build Vortex CI AMI" -description: "Build a custom Amazon Machine Image for Vortex CI runners" - -inputs: - arch: - description: "Target architecture: x64 or arm64" - required: true - ami-prefix: - description: "Prefix for AMI name" - required: false - default: "vortex-ci" - retention-days: - description: "Number of days before AMI is deprecated" - required: false - default: "30" - -outputs: - ami-id: - description: "The ID of the built AMI" - value: ${{ steps.create-ami.outputs.ami_id }} - ami-name: - description: "The name of the built AMI" - value: ${{ steps.create-ami.outputs.ami_name }} - -runs: - using: "composite" - steps: - - name: Setup Rust - uses: ./.github/actions/setup-rust - - - name: Setup flatc - uses: ./.github/actions/setup-flatc - - - name: Install extra dependencies - shell: bash - run: | - sudo apt-get update - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - cmake \ - ninja-build \ - clang \ - lld \ - llvm - - - name: Install nightly toolchain - shell: bash - run: | - rustup toolchain install nightly - rustup component add --toolchain nightly rustfmt clippy rust-src miri llvm-tools-preview - - - name: Install cargo tools - shell: bash - run: | - cargo install cargo-nextest --locked - cargo install cargo-hack --locked - cargo install grcov --locked - - - name: Clean runner for AMI - shell: bash - run: | - echo "=== Cleaning runner state for clean AMI ===" - - # The runner service name - RUNNER_SVC=$(systemctl list-units --type=service | grep actions.runner | awk '{print $1}' | head -1) - - if [ -n "$RUNNER_SVC" ]; then - echo "Stopping runner service: $RUNNER_SVC" - sudo systemctl stop "$RUNNER_SVC" || true - fi - - # Remove runner registration (this is the key!) - RUNNER_DIR="/home/runner/actions-runner" - if [ -d "$RUNNER_DIR" ]; then - echo "Cleaning runner directory..." - sudo rm -f "$RUNNER_DIR/.runner" || true - sudo rm -f "$RUNNER_DIR/.credentials" || true - sudo rm -f "$RUNNER_DIR/.credentials_rsaparams" || true - sudo rm -rf "$RUNNER_DIR/_diag" || true - sudo rm -rf "$RUNNER_DIR/_work" || true - fi - - # Clear temp files - sudo rm -rf /tmp/* || true - sudo rm -rf /var/tmp/* || true - - # Clear apt cache to reduce AMI size - sudo apt-get clean - sudo rm -rf /var/lib/apt/lists/* - - echo "=== Runner state cleaned ===" - - - name: Create AMI - id: create-ami - shell: bash - env: - AMI_PREFIX: ${{ inputs.ami-prefix }} - ARCH: ${{ inputs.arch }} - RETENTION_DAYS: ${{ inputs.retention-days }} - run: | - export AWS_REGION="$RUNS_ON_AWS_REGION" - export AWS_DEFAULT_REGION="$RUNS_ON_AWS_REGION" - - TIMESTAMP=$(date +%Y%m%d-%H%M%S) - AMI_NAME="${AMI_PREFIX}-${ARCH}-${TIMESTAMP}" - DEPRECATION_TIME=$(date -u -d "+${RETENTION_DAYS} days" +%Y-%m-%dT%H:%M:%SZ) - - echo "Creating AMI: $AMI_NAME from instance $RUNS_ON_INSTANCE_ID in region $AWS_REGION" - - AMI_ID=$(aws ec2 create-image \ - --instance-id "$RUNS_ON_INSTANCE_ID" \ - --name "$AMI_NAME" \ - --description "Vortex CI runner image for ${ARCH}" \ - --no-reboot \ - --tag-specifications "ResourceType=image,Tags=[{Key=Name,Value=$AMI_NAME},{Key=Environment,Value=ci},{Key=Arch,Value=$ARCH},{Key=ManagedBy,Value=github-actions}]" \ - --query 'ImageId' \ - --output text) - - echo "Waiting for AMI to be available..." - aws ec2 wait image-available --image-ids "$AMI_ID" - - echo "Setting deprecation time to $DEPRECATION_TIME" - aws ec2 enable-image-deprecation \ - --image-id "$AMI_ID" \ - --deprecate-at "$DEPRECATION_TIME" - - echo "ami_id=$AMI_ID" >> $GITHUB_OUTPUT - echo "ami_name=$AMI_NAME" >> $GITHUB_OUTPUT - echo "AMI created: $AMI_ID ($AMI_NAME)" diff --git a/.github/packer/vortex-ci.pkr.hcl b/.github/packer/vortex-ci.pkr.hcl index dbfaf37c7c3..7830fe4a9dc 100644 --- a/.github/packer/vortex-ci.pkr.hcl +++ b/.github/packer/vortex-ci.pkr.hcl @@ -33,6 +33,12 @@ variable "subnet_id" { default = "" } +variable "security_group_id" { + type = string + default = "" + description = "Existing security group ID (must allow SSH inbound)" +} + variable "rust_toolchain" { type = string default = "1.89" @@ -83,8 +89,9 @@ source "amazon-ebs" "vortex-ci" { owners = [var.source_ami_owner] } - subnet_id = var.subnet_id != "" ? var.subnet_id : null - ssh_username = "runner" + subnet_id = var.subnet_id != "" ? var.subnet_id : null + security_group_id = var.security_group_id != "" ? var.security_group_id : null + ssh_username = "runner" # User data to start SSH for Packer connectivity user_data_file = "${path.root}/scripts/user_data.sh" diff --git a/.github/workflows/ami-prebuild.yml b/.github/workflows/ami-prebuild.yml index 7abd416310a..884964130f6 100644 --- a/.github/workflows/ami-prebuild.yml +++ b/.github/workflows/ami-prebuild.yml @@ -40,19 +40,14 @@ env: AWS_REGION: eu-west-1 jobs: - # Build AMI by running on a runs-on instance and creating an image from it build-x64: name: "Build AMI (x64)" if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.arch == '' || github.event.inputs.arch == 'x64' }} - runs-on: - - runs-on=${{ github.run_id }} - - runner=2cpu-linux-x64 - - family=m7i+m7i-flex+m7a - - tag=ami-build-x64 + runs-on: ubuntu-latest timeout-minutes: 60 outputs: - ami-id: ${{ steps.build-ami.outputs.ami-id }} - ami-name: ${{ steps.build-ami.outputs.ami-name }} + ami-id: ${{ steps.build.outputs.ami_id }} + ami-name: ${{ steps.build.outputs.ami_name }} steps: - name: Checkout @@ -64,34 +59,57 @@ jobs: role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole aws-region: ${{ env.AWS_REGION }} - - name: Build AMI - id: build-ami - uses: ./.github/actions/build-ami + - name: Setup Packer + uses: hashicorp/setup-packer@main with: - arch: x64 - ami-prefix: vortex-ci - retention-days: ${{ inputs.retention-days || '30' }} + version: "1.11.2" + + - name: Packer Init + working-directory: .github/packer + run: packer init vortex-ci.pkr.hcl + + - name: Packer Build + id: build + working-directory: .github/packer + run: | + packer build \ + -var "arch=x64" \ + -var "aws_region=${{ env.AWS_REGION }}" \ + -var "subnet_id=${{ secrets.AWS_SUBNET_ID }}" \ + -var "security_group_id=${{ secrets.AWS_SECURITY_GROUP_ID }}" \ + -machine-readable \ + vortex-ci.pkr.hcl | tee packer-output.log + + # Extract AMI ID from Packer output + AMI_ID=$(grep 'artifact,0,id' packer-output.log | tail -1 | cut -d',' -f6 | cut -d':' -f2) + echo "ami_id=$AMI_ID" >> $GITHUB_OUTPUT + echo "ami_name=vortex-ci-x64" >> $GITHUB_OUTPUT + echo "Built AMI: $AMI_ID" + + - name: Set AMI Deprecation + run: | + RETENTION_DAYS=${{ inputs.retention-days || '30' }} + DEPRECATION_TIME=$(date -u -d "+${RETENTION_DAYS} days" +%Y-%m-%dT%H:%M:%SZ) + aws ec2 enable-image-deprecation \ + --image-id "${{ steps.build.outputs.ami_id }}" \ + --deprecate-at "$DEPRECATION_TIME" + echo "AMI will be deprecated at $DEPRECATION_TIME" - name: Summary run: | echo "## AMI Build Complete (x64)" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "- **AMI ID:** ${{ steps.build-ami.outputs.ami-id }}" >> $GITHUB_STEP_SUMMARY - echo "- **AMI Name:** ${{ steps.build-ami.outputs.ami-name }}" >> $GITHUB_STEP_SUMMARY + echo "- **AMI ID:** ${{ steps.build.outputs.ami_id }}" >> $GITHUB_STEP_SUMMARY echo "- **Deprecation:** ${{ inputs.retention-days || '30' }} days" >> $GITHUB_STEP_SUMMARY build-arm64: name: "Build AMI (arm64)" if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.arch == '' || github.event.inputs.arch == 'arm64' }} - runs-on: - - runs-on=${{ github.run_id }} - - runner=2cpu-linux-arm64 - - family=m7g - - tag=ami-build-arm64 + runs-on: ubuntu-latest timeout-minutes: 60 outputs: - ami-id: ${{ steps.build-ami.outputs.ami-id }} - ami-name: ${{ steps.build-ami.outputs.ami-name }} + ami-id: ${{ steps.build.outputs.ami_id }} + ami-name: ${{ steps.build.outputs.ami_name }} steps: - name: Checkout @@ -103,20 +121,47 @@ jobs: role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole aws-region: ${{ env.AWS_REGION }} - - name: Build AMI - id: build-ami - uses: ./.github/actions/build-ami + - name: Setup Packer + uses: hashicorp/setup-packer@main with: - arch: arm64 - ami-prefix: vortex-ci - retention-days: ${{ inputs.retention-days || '30' }} + version: "1.11.2" + + - name: Packer Init + working-directory: .github/packer + run: packer init vortex-ci.pkr.hcl + + - name: Packer Build + id: build + working-directory: .github/packer + run: | + packer build \ + -var "arch=arm64" \ + -var "aws_region=${{ env.AWS_REGION }}" \ + -var "subnet_id=${{ secrets.AWS_SUBNET_ID }}" \ + -var "security_group_id=${{ secrets.AWS_SECURITY_GROUP_ID }}" \ + -machine-readable \ + vortex-ci.pkr.hcl | tee packer-output.log + + # Extract AMI ID from Packer output + AMI_ID=$(grep 'artifact,0,id' packer-output.log | tail -1 | cut -d',' -f6 | cut -d':' -f2) + echo "ami_id=$AMI_ID" >> $GITHUB_OUTPUT + echo "ami_name=vortex-ci-arm64" >> $GITHUB_OUTPUT + echo "Built AMI: $AMI_ID" + + - name: Set AMI Deprecation + run: | + RETENTION_DAYS=${{ inputs.retention-days || '30' }} + DEPRECATION_TIME=$(date -u -d "+${RETENTION_DAYS} days" +%Y-%m-%dT%H:%M:%SZ) + aws ec2 enable-image-deprecation \ + --image-id "${{ steps.build.outputs.ami_id }}" \ + --deprecate-at "$DEPRECATION_TIME" + echo "AMI will be deprecated at $DEPRECATION_TIME" - name: Summary run: | echo "## AMI Build Complete (arm64)" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "- **AMI ID:** ${{ steps.build-ami.outputs.ami-id }}" >> $GITHUB_STEP_SUMMARY - echo "- **AMI Name:** ${{ steps.build-ami.outputs.ami-name }}" >> $GITHUB_STEP_SUMMARY + echo "- **AMI ID:** ${{ steps.build.outputs.ami_id }}" >> $GITHUB_STEP_SUMMARY echo "- **Deprecation:** ${{ inputs.retention-days || '30' }} days" >> $GITHUB_STEP_SUMMARY # Test the newly built AMI From 0b181cf9fbde8628fee253240b5165bb687af4d6 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 19 Dec 2025 13:41:20 +0000 Subject: [PATCH 11/11] Add debug job to list existing AMIs Signed-off-by: Joe Isaacs --- .github/workflows/ami-prebuild.yml | 35 ++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/.github/workflows/ami-prebuild.yml b/.github/workflows/ami-prebuild.yml index 884964130f6..6fe09ac0e80 100644 --- a/.github/workflows/ami-prebuild.yml +++ b/.github/workflows/ami-prebuild.yml @@ -40,6 +40,41 @@ env: AWS_REGION: eu-west-1 jobs: + debug-amis: + name: "Debug: List existing AMIs" + runs-on: ubuntu-latest + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole + aws-region: ${{ env.AWS_REGION }} + + - name: List vortex-ci AMIs + run: | + echo "=== x64 AMIs ===" + aws ec2 describe-images \ + --owners 375504701696 \ + --filters "Name=name,Values=vortex-ci-x64-*" \ + --query 'Images[*].[Name,ImageId,CreationDate,State]' \ + --output table || echo "No x64 AMIs found" + + echo "" + echo "=== arm64 AMIs ===" + aws ec2 describe-images \ + --owners 375504701696 \ + --filters "Name=name,Values=vortex-ci-arm64-*" \ + --query 'Images[*].[Name,ImageId,CreationDate,State]' \ + --output table || echo "No arm64 AMIs found" + + echo "" + echo "=== runs-on base AMIs (for reference) ===" + aws ec2 describe-images \ + --owners 135269210855 \ + --filters "Name=name,Values=runs-on-v2.2-ubuntu24-full-x64-*" \ + --query 'Images | sort_by(@, &CreationDate) | [-1].[Name,ImageId,CreationDate]' \ + --output table || echo "Cannot read runs-on AMIs" + build-x64: name: "Build AMI (x64)" if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.arch == '' || github.event.inputs.arch == 'x64' }}