diff --git a/.github/packer/scripts/provision.sh b/.github/packer/scripts/provision.sh new file mode 100755 index 00000000000..682917b6170 --- /dev/null +++ b/.github/packer/scripts/provision.sh @@ -0,0 +1,84 @@ +#!/bin/bash +set -euo pipefail + +# Variables (passed from Packer) +RUST_TOOLCHAIN="${RUST_TOOLCHAIN:-1.89}" +PROTOC_VERSION="${PROTOC_VERSION:-29.3}" +FLATC_VERSION="${FLATC_VERSION:-25.9.23}" + +echo "=== Installing Vortex CI dependencies ===" + +# Install build dependencies +echo "Installing system packages..." +sudo apt-get update +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + cmake \ + ninja-build \ + clang \ + lld \ + llvm \ + pkg-config \ + libssl-dev + +# Install Rust +echo "Installing Rust ${RUST_TOOLCHAIN}..." +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain "${RUST_TOOLCHAIN}" +source "$HOME/.cargo/env" +echo 'source $HOME/.cargo/env' >> "$HOME/.bashrc" + +# Install Rust components +rustup component add clippy rustfmt +rustup toolchain install nightly +rustup component add --toolchain nightly rustfmt clippy rust-src miri llvm-tools-preview + +echo "Rust installed:" +cargo --version +rustc --version + +# Install protoc +echo "Installing protoc ${PROTOC_VERSION}..." +ARCH=$(uname -m) +if [ "$ARCH" = "x86_64" ]; then + PROTOC_ARCH=linux-x86_64 +else + PROTOC_ARCH=linux-aarch_64 +fi +curl -fsSL -o /tmp/protoc.zip "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${PROTOC_ARCH}.zip" +sudo unzip -o /tmp/protoc.zip -d /usr/local bin/protoc 'include/*' +sudo chmod +x /usr/local/bin/protoc +rm /tmp/protoc.zip +protoc --version + +# Install flatc +echo "Installing flatc ${FLATC_VERSION}..." +if [ "$ARCH" = "x86_64" ]; then + curl -fsSL -o /tmp/flatc.zip "https://github.com/google/flatbuffers/releases/download/v${FLATC_VERSION}/Linux.flatc.binary.clang++-18.zip" + sudo unzip -o /tmp/flatc.zip -d /usr/local/bin + sudo chmod +x /usr/local/bin/flatc + rm /tmp/flatc.zip +else + # Build from source for ARM64 + git clone --depth 1 --branch "v${FLATC_VERSION}" https://github.com/google/flatbuffers.git /tmp/flatbuffers + cd /tmp/flatbuffers + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release . + ninja + sudo cp flatc /usr/local/bin/ + cd - + rm -rf /tmp/flatbuffers +fi +flatc --version + +# Install cargo tools +echo "Installing cargo tools..." +source "$HOME/.cargo/env" +cargo install cargo-nextest --locked +cargo install cargo-hack --locked +cargo install grcov --locked + +# Cleanup +echo "Cleaning up..." +sudo apt-get clean +sudo rm -rf /var/lib/apt/lists/* +rm -rf /tmp/* + +echo "=== Vortex CI dependencies installed successfully ===" diff --git a/.github/packer/scripts/user_data.sh b/.github/packer/scripts/user_data.sh new file mode 100755 index 00000000000..03e4aa50f7e --- /dev/null +++ b/.github/packer/scripts/user_data.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# Start SSH for Packer to connect +systemctl start ssh diff --git a/.github/packer/vortex-ci.pkr.hcl b/.github/packer/vortex-ci.pkr.hcl new file mode 100644 index 00000000000..7830fe4a9dc --- /dev/null +++ b/.github/packer/vortex-ci.pkr.hcl @@ -0,0 +1,127 @@ +packer { + required_plugins { + amazon = { + version = ">= 1.3.0" + source = "github.com/hashicorp/amazon" + } + } +} + +variable "aws_region" { + type = string + default = "eu-west-1" +} + +variable "arch" { + type = string + description = "Architecture: x64 or arm64" +} + +variable "ami_prefix" { + type = string + default = "vortex-ci" +} + +variable "source_ami_owner" { + type = string + default = "135269210855" + description = "runs-on AWS account ID" +} + +variable "subnet_id" { + type = string + default = "" +} + +variable "security_group_id" { + type = string + default = "" + description = "Existing security group ID (must allow SSH inbound)" +} + +variable "rust_toolchain" { + type = string + default = "1.89" +} + +variable "protoc_version" { + type = string + default = "29.3" +} + +variable "flatc_version" { + type = string + default = "25.9.23" +} + +locals { + timestamp = formatdate("YYYYMMDD-HHmmss", timestamp()) + + arch_config = { + x64 = { + instance_type = "m7i.large" + source_ami_name = "runs-on-v2.2-ubuntu24-full-x64-*" + ami_arch = "x86_64" + } + arm64 = { + instance_type = "m7g.large" + source_ami_name = "runs-on-v2.2-ubuntu24-full-arm64-*" + ami_arch = "arm64" + } + } + + config = local.arch_config[var.arch] +} + +source "amazon-ebs" "vortex-ci" { + ami_name = "${var.ami_prefix}-${var.arch}-${local.timestamp}" + instance_type = local.config.instance_type + region = var.aws_region + + source_ami_filter { + filters = { + name = local.config.source_ami_name + root-device-type = "ebs" + virtualization-type = "hvm" + architecture = local.config.ami_arch + } + most_recent = true + owners = [var.source_ami_owner] + } + + subnet_id = var.subnet_id != "" ? var.subnet_id : null + security_group_id = var.security_group_id != "" ? var.security_group_id : null + ssh_username = "runner" + + # User data to start SSH for Packer connectivity + user_data_file = "${path.root}/scripts/user_data.sh" + + launch_block_device_mappings { + device_name = "/dev/sda1" + volume_size = 80 + volume_type = "gp3" + delete_on_termination = true + } + + tags = { + Name = "${var.ami_prefix}-${var.arch}" + Environment = "ci" + Arch = var.arch + BuildTime = local.timestamp + ManagedBy = "packer" + } +} + +build { + sources = ["source.amazon-ebs.vortex-ci"] + + # Run the provisioning script + provisioner "shell" { + script = "${path.root}/scripts/provision.sh" + environment_vars = [ + "RUST_TOOLCHAIN=${var.rust_toolchain}", + "PROTOC_VERSION=${var.protoc_version}", + "FLATC_VERSION=${var.flatc_version}" + ] + } +} diff --git a/.github/runs-on.yml b/.github/runs-on.yml index 7970bf75e17..bad0ddf2b9f 100644 --- a/.github/runs-on.yml +++ b/.github/runs-on.yml @@ -1,11 +1,17 @@ +# Custom AMIs for Vortex CI runners +# These AMIs are automatically rebuilt every 15 days by the ami-prebuild.yml workflow +# to keep the GitHub Actions runner agent up to date (required to be <30 days old). +# +# AMI naming pattern: vortex-ci-{arch}-{timestamp} +# Built with: .github/actions/build-ami and .github/packer/vortex-ci.pkr.hcl images: vortex-ci-amd64: platform: "linux" arch: "x64" - name: "vortex-ci-*" + name: "vortex-ci-x64-*" owner: "375504701696" vortex-ci-arm64: platform: "linux" arch: "arm64" - name: "vortex-ci-*" + name: "vortex-ci-arm64-*" owner: "375504701696" diff --git a/.github/workflows/ami-prebuild.yml b/.github/workflows/ami-prebuild.yml new file mode 100644 index 00000000000..6fe09ac0e80 --- /dev/null +++ b/.github/workflows/ami-prebuild.yml @@ -0,0 +1,247 @@ +name: AMI Prebuild + +# Schedule to run every 15 days to keep runner agent up to date +# GitHub stops routing jobs to runners with agents older than 30 days +on: + # TODO: Remove push trigger after testing + push: + branches: [ji/ami-prebuild] + paths: + - ".github/workflows/ami-prebuild.yml" + - ".github/packer/**" + schedule: + # Run at 00:00 UTC on the 1st and 16th of each month (~15 days apart) + - cron: "0 0 1,16 * *" + workflow_dispatch: + inputs: + arch: + description: "Architecture to build (leave empty for both)" + required: false + type: choice + options: + - "" + - x64 + - arm64 + retention-days: + description: "Days until AMI deprecation" + required: false + type: number + default: 30 + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +permissions: + contents: read + id-token: write + +env: + AWS_REGION: eu-west-1 + +jobs: + debug-amis: + name: "Debug: List existing AMIs" + runs-on: ubuntu-latest + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole + aws-region: ${{ env.AWS_REGION }} + + - name: List vortex-ci AMIs + run: | + echo "=== x64 AMIs ===" + aws ec2 describe-images \ + --owners 375504701696 \ + --filters "Name=name,Values=vortex-ci-x64-*" \ + --query 'Images[*].[Name,ImageId,CreationDate,State]' \ + --output table || echo "No x64 AMIs found" + + echo "" + echo "=== arm64 AMIs ===" + aws ec2 describe-images \ + --owners 375504701696 \ + --filters "Name=name,Values=vortex-ci-arm64-*" \ + --query 'Images[*].[Name,ImageId,CreationDate,State]' \ + --output table || echo "No arm64 AMIs found" + + echo "" + echo "=== runs-on base AMIs (for reference) ===" + aws ec2 describe-images \ + --owners 135269210855 \ + --filters "Name=name,Values=runs-on-v2.2-ubuntu24-full-x64-*" \ + --query 'Images | sort_by(@, &CreationDate) | [-1].[Name,ImageId,CreationDate]' \ + --output table || echo "Cannot read runs-on AMIs" + + build-x64: + name: "Build AMI (x64)" + if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.arch == '' || github.event.inputs.arch == 'x64' }} + runs-on: ubuntu-latest + timeout-minutes: 60 + outputs: + ami-id: ${{ steps.build.outputs.ami_id }} + ami-name: ${{ steps.build.outputs.ami_name }} + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole + aws-region: ${{ env.AWS_REGION }} + + - name: Setup Packer + uses: hashicorp/setup-packer@main + with: + version: "1.11.2" + + - name: Packer Init + working-directory: .github/packer + run: packer init vortex-ci.pkr.hcl + + - name: Packer Build + id: build + working-directory: .github/packer + run: | + packer build \ + -var "arch=x64" \ + -var "aws_region=${{ env.AWS_REGION }}" \ + -var "subnet_id=${{ secrets.AWS_SUBNET_ID }}" \ + -var "security_group_id=${{ secrets.AWS_SECURITY_GROUP_ID }}" \ + -machine-readable \ + vortex-ci.pkr.hcl | tee packer-output.log + + # Extract AMI ID from Packer output + AMI_ID=$(grep 'artifact,0,id' packer-output.log | tail -1 | cut -d',' -f6 | cut -d':' -f2) + echo "ami_id=$AMI_ID" >> $GITHUB_OUTPUT + echo "ami_name=vortex-ci-x64" >> $GITHUB_OUTPUT + echo "Built AMI: $AMI_ID" + + - name: Set AMI Deprecation + run: | + RETENTION_DAYS=${{ inputs.retention-days || '30' }} + DEPRECATION_TIME=$(date -u -d "+${RETENTION_DAYS} days" +%Y-%m-%dT%H:%M:%SZ) + aws ec2 enable-image-deprecation \ + --image-id "${{ steps.build.outputs.ami_id }}" \ + --deprecate-at "$DEPRECATION_TIME" + echo "AMI will be deprecated at $DEPRECATION_TIME" + + - name: Summary + run: | + echo "## AMI Build Complete (x64)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- **AMI ID:** ${{ steps.build.outputs.ami_id }}" >> $GITHUB_STEP_SUMMARY + echo "- **Deprecation:** ${{ inputs.retention-days || '30' }} days" >> $GITHUB_STEP_SUMMARY + + build-arm64: + name: "Build AMI (arm64)" + if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.arch == '' || github.event.inputs.arch == 'arm64' }} + runs-on: ubuntu-latest + timeout-minutes: 60 + outputs: + ami-id: ${{ steps.build.outputs.ami_id }} + ami-name: ${{ steps.build.outputs.ami_name }} + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole + aws-region: ${{ env.AWS_REGION }} + + - name: Setup Packer + uses: hashicorp/setup-packer@main + with: + version: "1.11.2" + + - name: Packer Init + working-directory: .github/packer + run: packer init vortex-ci.pkr.hcl + + - name: Packer Build + id: build + working-directory: .github/packer + run: | + packer build \ + -var "arch=arm64" \ + -var "aws_region=${{ env.AWS_REGION }}" \ + -var "subnet_id=${{ secrets.AWS_SUBNET_ID }}" \ + -var "security_group_id=${{ secrets.AWS_SECURITY_GROUP_ID }}" \ + -machine-readable \ + vortex-ci.pkr.hcl | tee packer-output.log + + # Extract AMI ID from Packer output + AMI_ID=$(grep 'artifact,0,id' packer-output.log | tail -1 | cut -d',' -f6 | cut -d':' -f2) + echo "ami_id=$AMI_ID" >> $GITHUB_OUTPUT + echo "ami_name=vortex-ci-arm64" >> $GITHUB_OUTPUT + echo "Built AMI: $AMI_ID" + + - name: Set AMI Deprecation + run: | + RETENTION_DAYS=${{ inputs.retention-days || '30' }} + DEPRECATION_TIME=$(date -u -d "+${RETENTION_DAYS} days" +%Y-%m-%dT%H:%M:%SZ) + aws ec2 enable-image-deprecation \ + --image-id "${{ steps.build.outputs.ami_id }}" \ + --deprecate-at "$DEPRECATION_TIME" + echo "AMI will be deprecated at $DEPRECATION_TIME" + + - name: Summary + run: | + echo "## AMI Build Complete (arm64)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- **AMI ID:** ${{ steps.build.outputs.ami_id }}" >> $GITHUB_STEP_SUMMARY + echo "- **Deprecation:** ${{ inputs.retention-days || '30' }} days" >> $GITHUB_STEP_SUMMARY + + # Test the newly built AMI + test-x64: + name: "Test AMI (x64)" + needs: build-x64 + runs-on: + - runs-on=${{ github.run_id }} + - family=m7i+m7i-flex+m7a + - cpu=4 + - image=vortex-ci-amd64 + - tag=test-ami-x64 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + - name: Verify tools are installed + run: | + echo "Checking installed tools..." + cargo --version + rustc --version + rustup show + protoc --version + flatc --version + - name: Run cargo check + run: cargo check --locked + + test-arm64: + name: "Test AMI (arm64)" + needs: build-arm64 + runs-on: + - runs-on=${{ github.run_id }} + - family=m7g + - cpu=4 + - image=vortex-ci-arm64 + - tag=test-ami-arm64 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + - name: Verify tools are installed + run: | + echo "Checking installed tools..." + cargo --version + rustc --version + rustup show + protoc --version + flatc --version + - name: Run cargo check + run: cargo check --locked diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ad48e0efd5..45203f89f85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -273,10 +273,6 @@ jobs: with: sccache: s3 - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-rust - with: - toolchain: nightly - repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Rust Lint - Format run: cargo +nightly fmt --all --check - name: Rustc check @@ -295,7 +291,7 @@ jobs: - runs-on=${{ github.run_id }} - family=m7i+m7i-flex+m7a - cpu=16 - - image=ubuntu24-full-x64 + - image=vortex-ci-amd64 - extras=s3-cache - tag=rust-lint-no-default steps: