From 7f96fac5bfed7d8019f0739b8b63f7ca4b9be2af Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Thu, 7 May 2026 15:55:43 -0400 Subject: [PATCH 1/3] Add .git to list of manifest-ignored files because in git worktrees, it's a file, not a directory. --- rakelib/check_manifest.rake | 1 + 1 file changed, 1 insertion(+) diff --git a/rakelib/check_manifest.rake b/rakelib/check_manifest.rake index 4c53b963ec..af122b2cec 100644 --- a/rakelib/check_manifest.rake +++ b/rakelib/check_manifest.rake @@ -35,6 +35,7 @@ task check_manifest: :templates do ] ignore_files = %w[ + .git # in worktrees this is a file, not a directory .ruby-version .editorconfig .git-blame-ignore-revs From 27490db01ae517448db6bee24b17b9c9827d464c Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Fri, 17 Apr 2026 10:36:22 -0400 Subject: [PATCH 2/3] Gemfile: pin nokogiri to main on aarch64-mingw-ucrt sparklemotion/nokogiri#3530 is merged fixing the loader config on windows-11-arm, but has not yet landed in a release. Pin to `main` on the affected platform only, to unblock CI until a fixed nokogiri release ships. --- Gemfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Gemfile b/Gemfile index abcb824ddb..d4ce4ab2f7 100644 --- a/Gemfile +++ b/Gemfile @@ -19,3 +19,9 @@ platforms :mri, :windows do end gem "onigmo", platforms: :ruby + +# Until a nokogiri release includes sparklemotion/nokogiri#3530, aarch64-mingw-ucrt +# source-builds of libxml2 fail in libtool. Pin to main on that platform only. +if RUBY_PLATFORM =~ /aarch64.*mingw/ + gem "nokogiri", github: "sparklemotion/nokogiri", branch: "main" +end From 15d8e4d525621e09c8baff6e482a4af91703d0f3 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Thu, 12 Mar 2026 15:22:20 -0400 Subject: [PATCH 3/3] Add native gem precompilation support Package precompiled native gems for Ruby 3.3, 3.4, and 4.0 across 10 platforms: aarch64-linux-gnu, aarch64-linux-musl, aarch64-mingw-ucrt, arm-linux-gnu, arm-linux-musl, arm64-darwin, x64-mingw-ucrt, x86_64-darwin, x86_64-linux-gnu, and x86_64-linux-musl. - Add Ruby-version-aware extension loader in lib/prism.rb with GLIBC error message for musl/glibc mismatches - Add cross-compilation support to Rakefile using rake-compiler-dock - Add a reusable `build-gems.yml` workflow called by both the CI workflow (`native-gem-precompilation.yml`) and the publish workflow, building source and native gems across all platforms and Ruby versions - Add scripts for building, verifying, and testing gems Update `publish-gem.yml` to handle multi-gem publishing: `rubygems/release-gem` only supports a single gem, so this uses `rubygems/configure-rubygems-credentials` and loops `gem push` over each built `.gem`. Add a `workflow_dispatch` trigger alongside the existing tag-push trigger, and create/update a GitHub release for the tag with all gems plus a `CHECKSUMS.txt` manifest. Move testing of the "ruby" platform gem (compile-from-source) out of `main.yml` into `test-gems.yml` so we're testing the gem built by the workflow tooling. Pin all GitHub Actions to commit SHAs and apply zizmor-style hardening across the new and modified workflows. --- .github/scripts/gem-build | 45 ++++ .github/scripts/gem-install-test | 25 +++ .github/workflows/build-gems.yml | 103 ++++++++++ .github/workflows/main.yml | 76 ------- .github/workflows/publish-gem.yml | 68 ++++++- .github/workflows/test-gems.yml | 188 +++++++++++++++++ .gitignore | 1 + Gemfile | 1 + Gemfile.lock | 26 +++ Rakefile | 58 +++++- bin/build-gems | 53 +++++ docs/releasing.md | 73 ++++++- lib/prism.rb | 25 ++- rakelib/gemspec.rake | 22 ++ test/prism/packaging/test-gem-file-contents | 215 ++++++++++++++++++++ 15 files changed, 886 insertions(+), 93 deletions(-) create mode 100755 .github/scripts/gem-build create mode 100755 .github/scripts/gem-install-test create mode 100644 .github/workflows/build-gems.yml create mode 100644 .github/workflows/test-gems.yml create mode 100755 bin/build-gems create mode 100644 rakelib/gemspec.rake create mode 100755 test/prism/packaging/test-gem-file-contents diff --git a/.github/scripts/gem-build b/.github/scripts/gem-build new file mode 100755 index 0000000000..f80d1aa02f --- /dev/null +++ b/.github/scripts/gem-build @@ -0,0 +1,45 @@ +#! /usr/bin/env bash +# +# run as part of CI, or as part of the release pipeline with --release +# +RELEASE=false +if [[ "${1:-}" == "--release" ]] ; then + RELEASE=true + shift +fi + +if [[ $# -lt 2 ]] ; then + echo "usage: $(basename $0) [--release] " + exit 1 +fi + +set -e -u + +OUTPUT_DIR=$1 +BUILD_NATIVE_GEM=$2 + +test -e /etc/os-release && cat /etc/os-release + +set -x + +bundle config set without development + +if [[ "${RELEASE}" == "true" ]] ; then + bundle install --local || bundle install +else + bundle config set frozen false + bundle install --local || bundle install + bundle exec rake gemspec:fake-version +fi + +if [[ "${BUILD_NATIVE_GEM}" == "ruby" ]] ; then + bundle exec rake build +else + bundle exec rake gem:${BUILD_NATIVE_GEM}:build +fi + +./test/prism/packaging/test-gem-file-contents pkg/*.gem + +mkdir -p ${OUTPUT_DIR} +cp -v pkg/*.gem ${OUTPUT_DIR} +ls -l ${OUTPUT_DIR}/* diff --git a/.github/scripts/gem-install-test b/.github/scripts/gem-install-test new file mode 100755 index 0000000000..e3999d3f55 --- /dev/null +++ b/.github/scripts/gem-install-test @@ -0,0 +1,25 @@ +#! /usr/bin/env sh +# +# run as part of CI +# +if [ $# -lt 1 ] ; then + echo "usage: $(basename $0) " + exit 1 +fi + +gemfile=$1 +shift + +test -e /etc/os-release && cat /etc/os-release + +set -e -x -u + +ls -l ${gemfile} +gem install --no-document ${gemfile} +gem list -d prism + +bundle config set without development +bundle install --local || bundle install + +rm -rf lib ext # ensure we don't use the local files +rake test diff --git a/.github/workflows/build-gems.yml b/.github/workflows/build-gems.yml new file mode 100644 index 0000000000..479a6b8154 --- /dev/null +++ b/.github/workflows/build-gems.yml @@ -0,0 +1,103 @@ +name: Build gems +on: + workflow_call: + inputs: + release: + description: "Pass --release to .github/scripts/gem-build (skip timestamp version)" + type: boolean + default: false + artifact_prefix: + description: "Prefix for artifact names (e.g., 'release-v1.10.0-')" + type: string + default: "" + ref: + description: "Git ref to check out (e.g., a version tag). Defaults to the caller's ref." + type: string + default: "" + +permissions: + contents: read + +jobs: + build_source_gem: + name: "source gem" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.ref || github.ref }} + persist-credentials: false + - uses: ruby/setup-ruby@c4e5b1316158f92e3d49443a9d58b31d25ac0f8f # v1.306.0 + with: + ruby-version: "3.4" + bundler-cache: true + - name: Build source gem + run: | + # shellcheck disable=SC2086 # RELEASE_FLAG intentionally word-splits (empty or "--release") + ./.github/scripts/gem-build ${RELEASE_FLAG} gems ruby + env: + RELEASE_FLAG: ${{ inputs.release && '--release' || '' }} + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: "${{ inputs.artifact_prefix }}source-gem" + path: gems + retention-days: 1 + + build_native_setup: + name: "setup" + runs-on: ubuntu-latest + outputs: + rcd_image_version: ${{ steps.rcd_image_version.outputs.rcd_image_version }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.ref || github.ref }} + persist-credentials: false + - uses: ruby/setup-ruby@c4e5b1316158f92e3d49443a9d58b31d25ac0f8f # v1.306.0 + with: + ruby-version: "3.4" + bundler-cache: true + - id: rcd_image_version + run: bundle exec ruby -e 'require "rake_compiler_dock"; puts "rcd_image_version=#{RakeCompilerDock::IMAGE_VERSION}"' >> "$GITHUB_OUTPUT" + + build_native_gem: + needs: build_native_setup + name: "native gem ${{ matrix.platform }}" + strategy: + # CI runs want to surface every platform's failure (fail-fast: false); + # release runs should stop on first failure to avoid pushing a partial + # set of gems. + fail-fast: ${{ inputs.release }} + matrix: + platform: + - aarch64-linux-gnu + - aarch64-linux-musl + - aarch64-mingw-ucrt + - arm-linux-gnu + - arm-linux-musl + - arm64-darwin + - x64-mingw-ucrt + - x86_64-darwin + - x86_64-linux-gnu + - x86_64-linux-musl + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.ref || github.ref }} + persist-credentials: false + - name: Build native gem + run: | + # shellcheck disable=SC2086 # RELEASE_FLAG intentionally word-splits (empty or "--release") + docker run --rm -v "$PWD:/work" -w /work \ + "ghcr.io/rake-compiler/rake-compiler-dock-image:${RCD_IMAGE_VERSION}-mri-${PLATFORM}" \ + ./.github/scripts/gem-build ${RELEASE_FLAG} gems "${PLATFORM}" + env: + RCD_IMAGE_VERSION: ${{ needs.build_native_setup.outputs.rcd_image_version }} + RELEASE_FLAG: ${{ inputs.release && '--release' || '' }} + PLATFORM: ${{ matrix.platform }} + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: "${{ inputs.artifact_prefix }}cruby-${{ matrix.platform }}-gem" + path: gems + retention-days: 1 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 24a76ff250..de2bea0b37 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -272,82 +272,6 @@ jobs: - name: Run Ruby tests with valgrind run: bundle exec rake test:valgrind - gem-package: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: ruby/setup-ruby@v1 - with: - ruby-version: head - bundler-cache: true - - run: bundle config --local frozen false - - run: bundle exec rake build:dev - - uses: actions/upload-artifact@v7 - with: - name: gem-package - path: pkg - retention-days: 1 - - gem-install: - needs: ["gem-package"] - strategy: - fail-fast: false - matrix: - target: - - { ruby: "2.7", os: "ubuntu-latest", gemfile: "2.7" } - - { ruby: "3.0", os: "ubuntu-latest", gemfile: "3.0" } - - { ruby: "3.1", os: "ubuntu-latest", gemfile: "3.1" } - - { ruby: "3.2", os: "ubuntu-latest", gemfile: "3.2" } - - { ruby: "3.3", os: "ubuntu-latest", gemfile: "3.3" } - - { ruby: "3.4", os: "ubuntu-latest", gemfile: "3.4" } - - { ruby: "4.0", os: "ubuntu-latest", gemfile: "4.0" } - - { ruby: "head", os: "ubuntu-latest", gemfile: "4.1" } - - { ruby: "jruby", os: "ubuntu-latest", gemfile: ".." } - - { ruby: "truffleruby", os: "ubuntu-latest", gemfile: ".." } - - - { ruby: "2.7", os: "macos-latest", gemfile: "2.7" } - - { ruby: "3.0", os: "macos-latest", gemfile: "3.0" } - - { ruby: "3.1", os: "macos-latest", gemfile: "3.1" } - - { ruby: "3.2", os: "macos-latest", gemfile: "3.2" } - - { ruby: "3.3", os: "macos-latest", gemfile: "3.3" } - - { ruby: "3.4", os: "macos-latest", gemfile: "3.4" } - - { ruby: "4.0", os: "macos-latest", gemfile: "4.0" } - - { ruby: "head", os: "macos-latest", gemfile: "4.1" } - - { ruby: "jruby", os: "macos-latest", gemfile: ".." } - - { ruby: "truffleruby", os: "macos-latest", gemfile: ".." } - - - { ruby: "2.7", os: "windows-latest", gemfile: "2.7" } - - { ruby: "3.0", os: "windows-latest", gemfile: "3.0" } - - { ruby: "3.1", os: "windows-latest", gemfile: "3.1" } - - { ruby: "3.2", os: "windows-latest", gemfile: "3.2" } - - { ruby: "3.3", os: "windows-latest", gemfile: "3.3" } - - { ruby: "3.4", os: "windows-latest", gemfile: "3.4" } - - { ruby: "4.0", os: "windows-latest", gemfile: "4.0" } - - { ruby: "head", os: "windows-latest", gemfile: "4.1" } - - { ruby: "jruby", os: "windows-latest", gemfile: ".." } - env: - BUNDLE_GEMFILE: gemfiles/${{ matrix.target.gemfile }}/Gemfile - runs-on: ${{ matrix.target.os }} - steps: - - uses: actions/checkout@v6 - - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ matrix.target.ruby }} - - uses: actions/download-artifact@v8 - with: - name: gem-package - path: pkg - - run: | - gem install --local pkg/prism-*.gem - gem list -d prism - shell: bash - - name: Run tests - run: | - bundle install - rm -rf lib ext # ensure we don't use the local files - rake test - shell: bash - gcc-analyzer: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/publish-gem.yml b/.github/workflows/publish-gem.yml index 87d6cf5657..b426de3f91 100644 --- a/.github/workflows/publish-gem.yml +++ b/.github/workflows/publish-gem.yml @@ -1,39 +1,87 @@ name: Publish gem to rubygems.org +concurrency: + group: "release-${{ inputs.version_tag || github.ref }}" + cancel-in-progress: false on: push: tags: - 'v*' + workflow_dispatch: + inputs: + version_tag: + description: "Version tag to release (e.g., v1.10.0)" + required: true + type: string permissions: contents: read jobs: + build: + uses: ./.github/workflows/build-gems.yml + with: + release: true + artifact_prefix: "release-${{ inputs.version_tag || github.ref_name }}-" + ref: ${{ inputs.version_tag || github.ref }} + push: + name: "Push gems and create GitHub release" + needs: build if: github.repository == 'ruby/prism' runs-on: ubuntu-latest - environment: name: rubygems.org url: https://rubygems.org/gems/prism - permissions: - contents: write - id-token: write - + contents: write # create/update GitHub releases and upload .gem assets + id-token: write # OIDC token for rubygems.org trusted publishing steps: - name: Harden Runner - uses: step-security/harden-runner@v2 + uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1 with: egress-policy: audit - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.version_tag || github.ref }} + persist-credentials: false - name: Set up Ruby - uses: ruby/setup-ruby@v1 + uses: ruby/setup-ruby@c4e5b1316158f92e3d49443a9d58b31d25ac0f8f # v1.306.0 # zizmor: ignore[cache-poisoning] -- cache is for gem-push tooling only; release artifacts come from prior job's uploaded artifacts, not this Ruby env with: ruby-version: "3.4" bundler-cache: true - - name: Publish to RubyGems - uses: rubygems/release-gem@v1 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: gems + pattern: "release-${{ inputs.version_tag || github.ref_name }}-*" + merge-multiple: true + + - name: Print and record checksums + run: | + cd gems + sha256sum ./*.gem | tee CHECKSUMS.txt + + - uses: rubygems/configure-rubygems-credentials@762a4b77c3300434bb57c7ce80b20e36231927aa # v2.0.0 + + # TODO: once RubyGems >= 4.1 is in setup-ruby, `gem push` will auto-attest (ruby/rubygems#9325). + - name: Push gems to RubyGems.org + run: | + for gem in gems/*.gem; do + echo "Pushing ${gem} ..." + gem push "${gem}" || echo "WARNING: Failed to push ${gem} (may already exist)" + done + + - name: Create or update GitHub release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ inputs.version_tag || github.ref_name }} + REPO: ${{ github.repository }} + run: | + if gh release view "$TAG" --repo "$REPO" >/dev/null 2>&1 ; then + gh release upload "$TAG" --repo "$REPO" --clobber gems/*.gem gems/CHECKSUMS.txt + else + gh release create "$TAG" --repo "$REPO" --title "$TAG" --generate-notes gems/*.gem gems/CHECKSUMS.txt + fi diff --git a/.github/workflows/test-gems.yml b/.github/workflows/test-gems.yml new file mode 100644 index 0000000000..0b7cd04dd1 --- /dev/null +++ b/.github/workflows/test-gems.yml @@ -0,0 +1,188 @@ +name: Test gems +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: true + +on: + push: + branches: + - main + tags: + - v* + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + uses: ./.github/workflows/build-gems.yml + + install_source_gem: + needs: build + name: "install source gem ${{ matrix.target.os }} ${{ matrix.target.ruby }}" + strategy: + fail-fast: false + matrix: + target: + - { ruby: "2.7", os: "ubuntu-latest", gemfile: "2.7" } + - { ruby: "3.0", os: "ubuntu-latest", gemfile: "3.0" } + - { ruby: "3.1", os: "ubuntu-latest", gemfile: "3.1" } + - { ruby: "3.2", os: "ubuntu-latest", gemfile: "3.2" } + - { ruby: "3.3", os: "ubuntu-latest", gemfile: "3.3" } + - { ruby: "3.4", os: "ubuntu-latest", gemfile: "3.4" } + - { ruby: "4.0", os: "ubuntu-latest", gemfile: "4.0" } + - { ruby: "head", os: "ubuntu-latest", gemfile: "4.1" } + - { ruby: "jruby", os: "ubuntu-latest", gemfile: ".." } + - { ruby: "truffleruby", os: "ubuntu-latest", gemfile: ".." } + + - { ruby: "2.7", os: "macos-latest", gemfile: "2.7" } + - { ruby: "3.0", os: "macos-latest", gemfile: "3.0" } + - { ruby: "3.1", os: "macos-latest", gemfile: "3.1" } + - { ruby: "3.2", os: "macos-latest", gemfile: "3.2" } + - { ruby: "3.3", os: "macos-latest", gemfile: "3.3" } + - { ruby: "3.4", os: "macos-latest", gemfile: "3.4" } + - { ruby: "4.0", os: "macos-latest", gemfile: "4.0" } + - { ruby: "head", os: "macos-latest", gemfile: "4.1" } + - { ruby: "jruby", os: "macos-latest", gemfile: ".." } + - { ruby: "truffleruby", os: "macos-latest", gemfile: ".." } + + - { ruby: "2.7", os: "windows-latest", gemfile: "2.7" } + - { ruby: "3.0", os: "windows-latest", gemfile: "3.0" } + - { ruby: "3.1", os: "windows-latest", gemfile: "3.1" } + - { ruby: "3.2", os: "windows-latest", gemfile: "3.2" } + - { ruby: "3.3", os: "windows-latest", gemfile: "3.3" } + - { ruby: "3.4", os: "windows-latest", gemfile: "3.4" } + - { ruby: "4.0", os: "windows-latest", gemfile: "4.0" } + - { ruby: "head", os: "windows-latest", gemfile: "4.1" } + - { ruby: "jruby", os: "windows-latest", gemfile: ".." } + env: + BUNDLE_GEMFILE: gemfiles/${{ matrix.target.gemfile }}/Gemfile + runs-on: ${{ matrix.target.os }} + steps: + - if: startsWith(matrix.target.os, 'windows') + name: configure git crlf + run: | + git config --system core.autocrlf false + git config --system core.eol lf + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: ruby/setup-ruby@c4e5b1316158f92e3d49443a9d58b31d25ac0f8f # v1.306.0 + with: + ruby-version: ${{ matrix.target.ruby }} + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: source-gem + path: gems + - run: ./.github/scripts/gem-install-test gems/*.gem + shell: sh + + test_linux_native: + name: "install native gem ${{ matrix.platform }} ${{ matrix.ruby }}" + needs: build + strategy: + fail-fast: false + matrix: + platform: + - aarch64-linux-gnu + - aarch64-linux-musl + - arm-linux-gnu + - arm-linux-musl + - x86_64-linux-gnu + - x86_64-linux-musl + ruby: ["3.3", "3.4", "4.0"] + include: + # musl platforms need alpine image and build-base + - { platform: aarch64-linux-musl, docker_tag: "-alpine", bootstrap: "apk add build-base linux-headers yaml-dev &&" } + - { platform: arm-linux-musl, docker_tag: "-alpine", bootstrap: "apk add build-base linux-headers yaml-dev &&" } + - { platform: x86_64-linux-musl, docker_tag: "-alpine", bootstrap: "apk add build-base linux-headers yaml-dev &&" } + # docker platform for each platform + - { platform: aarch64-linux-gnu, runner: ubuntu-24.04-arm, docker_platform: "--platform=linux/arm64" } + - { platform: aarch64-linux-musl, runner: ubuntu-24.04-arm, docker_platform: "--platform=linux/arm64" } + - { platform: arm-linux-gnu, runner: ubuntu-24.04-arm, docker_platform: "--platform=linux/arm/v7" } + - { platform: arm-linux-musl, runner: ubuntu-24.04-arm, docker_platform: "--platform=linux/arm/v7" } + runs-on: ${{ matrix.runner || 'ubuntu-latest' }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: cruby-${{ matrix.platform }}-gem + path: gems + - name: Install and test gem + env: + DOCKER_PLATFORM: ${{ matrix.docker_platform }} + RUBY_VERSION: ${{ matrix.ruby }} + DOCKER_TAG: ${{ matrix.docker_tag }} + BOOTSTRAP: ${{ matrix.bootstrap }} + run: | + # shellcheck disable=SC2086 # DOCKER_PLATFORM intentionally word-splits (empty or "--platform=...") + docker run --rm -v "$PWD:/work" -w /work \ + ${DOCKER_PLATFORM} "ruby:${RUBY_VERSION}${DOCKER_TAG}" \ + sh -c " + ${BOOTSTRAP} + ./.github/scripts/gem-install-test gems/*.gem + " + + test_darwin_native: + name: "install native gem ${{ matrix.platform }} ${{ matrix.ruby }}" + needs: build + strategy: + fail-fast: false + matrix: + os: [macos-15, macos-15-intel] + ruby: ["3.3", "3.4", "4.0"] + include: + - os: macos-15 + platform: arm64-darwin + - os: macos-15-intel + platform: x86_64-darwin + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: ruby/setup-ruby@c4e5b1316158f92e3d49443a9d58b31d25ac0f8f # v1.306.0 + with: + ruby-version: "${{ matrix.ruby }}" + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: cruby-${{ matrix.platform }}-gem + path: gems + - run: ./.github/scripts/gem-install-test gems/*.gem + shell: sh + + test_windows_native: + name: "install native gem ${{ matrix.platform }} ${{ matrix.ruby }}" + needs: build + strategy: + fail-fast: false + matrix: + platform: [x64-mingw-ucrt, aarch64-mingw-ucrt] + ruby: ["3.3", "3.4", "4.0"] + include: + - { platform: x64-mingw-ucrt, os: windows-latest } + - { platform: aarch64-mingw-ucrt, os: windows-11-arm } + exclude: + - { platform: aarch64-mingw-ucrt, ruby: "3.3" } + runs-on: ${{ matrix.os }} + steps: + - name: configure git crlf + run: | + git config --system core.autocrlf false + git config --system core.eol lf + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: ruby/setup-ruby@c4e5b1316158f92e3d49443a9d58b31d25ac0f8f # v1.306.0 + with: + ruby-version: "${{ matrix.ruby }}" + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: cruby-${{ matrix.platform }}-gem + path: gems + - run: ./.github/scripts/gem-install-test gems/*.gem + shell: sh diff --git a/.gitignore b/.gitignore index 0447b6ef70..dd130e4f00 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ tmp/ vendor/ /build/ +/gems/ /lib/prism/prism.* /lib/prism.bundle /lib/prism.so diff --git a/Gemfile b/Gemfile index d4ce4ab2f7..dad7c06177 100644 --- a/Gemfile +++ b/Gemfile @@ -8,6 +8,7 @@ gem "benchmark-ips" gem "parser" gem "rake" gem "rake-compiler" +gem "rake-compiler-dock", "~> 1.12.0" gem "ruby_parser" gem "test-unit" diff --git a/Gemfile.lock b/Gemfile.lock index f42c601c6f..6c738e2e7e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -13,6 +13,12 @@ GEM erb (6.0.2) erb (6.0.2-java) ffi (1.17.3) + ffi (1.17.3-aarch64-linux-gnu) + ffi (1.17.3-aarch64-linux-musl) + ffi (1.17.3-arm64-darwin) + ffi (1.17.3-x86_64-darwin) + ffi (1.17.3-x86_64-linux-gnu) + ffi (1.17.3-x86_64-linux-musl) io-console (0.8.2) io-console (0.8.2-java) irb (1.17.0) @@ -25,8 +31,20 @@ GEM nokogiri (1.19.1) mini_portile2 (~> 2.8.2) racc (~> 1.4) + nokogiri (1.19.1-aarch64-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.1-aarch64-linux-musl) + racc (~> 1.4) + nokogiri (1.19.1-arm64-darwin) + racc (~> 1.4) nokogiri (1.19.1-java) racc (~> 1.4) + nokogiri (1.19.1-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.19.1-x86_64-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.1-x86_64-linux-musl) + racc (~> 1.4) onigmo (0.1.0) parser (3.3.10.2) ast (~> 2.4.1) @@ -46,6 +64,7 @@ GEM rake (13.3.1) rake-compiler (1.3.1) rake + rake-compiler-dock (1.12.0) rdoc (7.2.0) erb psych (>= 4.0.0) @@ -64,8 +83,14 @@ GEM tsort (0.2.0) PLATFORMS + aarch64-linux-gnu + aarch64-linux-musl + arm64-darwin java ruby + x86_64-darwin + x86_64-linux-gnu + x86_64-linux-musl DEPENDENCIES benchmark-ips @@ -76,6 +101,7 @@ DEPENDENCIES prism! rake rake-compiler + rake-compiler-dock (~> 1.12.0) rdoc ruby_memcheck ruby_parser diff --git a/Rakefile b/Rakefile index 421a4df697..fd37486dbd 100644 --- a/Rakefile +++ b/Rakefile @@ -3,6 +3,15 @@ require "bundler/gem_tasks" require "rake/clean" +begin + require "rake_compiler_dock" +rescue LoadError + # rake_compiler_dock is not available in versioned Gemfiles (gemfiles/2.7/, etc.) + # that don't include the gem. The native gem tasks will not be defined. +end + +PRISM_SPEC = Bundler.load_gemspec("prism.gemspec") + task default: %i[compile test] require_relative "templates/template" @@ -28,14 +37,55 @@ task build: [:check_manifest, :templates] # the C extension task "compile:prism" => ["templates"] # must be before the ExtensionTask is created +RCD_CROSS_PLATFORMS = %w[ + aarch64-linux-gnu + aarch64-linux-musl + aarch64-mingw-ucrt + arm-linux-gnu + arm-linux-musl + arm64-darwin + x64-mingw-ucrt + x86_64-darwin + x86_64-linux-gnu + x86_64-linux-musl +] + +if defined?(RakeCompilerDock) + RakeCompilerDock.set_ruby_cc_version(">= 3.3") + + namespace "gem" do + RCD_CROSS_PLATFORMS.each do |platform| + desc "build native gem for #{platform}" + task platform do + RakeCompilerDock.sh(<<~EOF, platform: platform, verbose: true) + gem install bundler --no-document && + bundle && + bundle exec rake gem:#{platform}:build + EOF + end + + namespace platform do + # this runs in the rake-compiler-dock docker container + task "build" => ["templates"] do + Rake::Task["native:#{platform}"].invoke + Rake::Task["pkg/#{PRISM_SPEC.full_name}-#{Gem::Platform.new(platform)}.gem"].invoke + end + end + end + + desc "build native gem for all platforms" + task "all" => [RCD_CROSS_PLATFORMS, "build"].flatten + end +end + if RUBY_ENGINE == "ruby" and !ENV["PRISM_FFI_BACKEND"] require "rake/extensiontask" - Rake::ExtensionTask.new(:compile) do |ext| - ext.name = "prism" + Rake::ExtensionTask.new("prism", PRISM_SPEC) do |ext| ext.ext_dir = "ext/prism" ext.lib_dir = "lib/prism" - ext.gem_spec = Gem::Specification.load("prism.gemspec") + ext.cross_compile = true + ext.cross_platform = RCD_CROSS_PLATFORMS end elsif RUBY_ENGINE == "jruby" require "rake/javaextensiontask" @@ -46,7 +96,7 @@ elsif RUBY_ENGINE == "jruby" ext.ext_dir = "java/api" ext.lib_dir = "tmp" ext.release = "21" - ext.gem_spec = Gem::Specification.load("prism.gemspec") + ext.gem_spec = PRISM_SPEC end end diff --git a/bin/build-gems b/bin/build-gems new file mode 100755 index 0000000000..af9babbd03 --- /dev/null +++ b/bin/build-gems @@ -0,0 +1,53 @@ +#! /usr/bin/env bash +# +# script to build gems for all relevant platforms +# +set -o errexit +set -o nounset +set -x + +rm -rf tmp pkg gems +mkdir -p gems + +# prelude: vendor dependencies +bundle update +bundle package + +# safety check: let's check that things work +bundle exec rake clean clobber +bundle exec rake compile test + +# package the gems, including precompiled native +bundle exec rake clean clobber +bundle exec rake gem:all +cp -v pkg/prism*.gem gems + +# test those gem files! +for gemfile in gems/*.gem ; do + ./test/prism/packaging/test-gem-file-contents "$gemfile" +done + +# smoke test: install and load the native gem for the host platform +host_gem=$(ruby -e ' + platform = Gem::Platform.local + # normalize: remove version from platform string (e.g., "x86_64-linux-gnu" not "x86_64-linux-gnu6.2") + normalized = [platform.cpu, platform.os, platform.version].compact.join("-") + gem = Dir.glob("gems/prism-*-#{normalized}.gem").first + gem ||= Dir.glob("gems/prism-*-#{platform.cpu}-#{platform.os}*.gem").first + puts gem.to_s +') +if [ -n "$host_gem" ] ; then + echo "--- smoke test: install and load $host_gem ---" + test_dir=$(mktemp -d) + GEM_HOME="$test_dir" GEM_PATH="$test_dir" gem install --no-document "$host_gem" + GEM_HOME="$test_dir" GEM_PATH="$test_dir" ruby -e 'require "prism"; Prism.parse("1 + 2"); puts "smoke test passed: #{Prism::VERSION}"' + rm -rf "$test_dir" + echo "--- smoke test passed ---" +else + echo "--- skipping install smoke test (no native gem for this host platform) ---" +fi + +# checksums should be included in the release notes +pushd gems + sha256sum *.gem +popd diff --git a/docs/releasing.md b/docs/releasing.md index 6d0f892926..1afb0f2958 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -71,5 +71,74 @@ git push ## Publishing -* Update the GitHub release page with a copy of the latest entry in the `CHANGELOG.md` file. -* Push a new tag to the GitHub repository, following the `vX.Y.Z` format. +### Automatic (Github Actions) + +The [`publish-gem.yml`](../.github/workflows/publish-gem.yml) workflow handles publishing. It triggers either on a `v*` tag push or via manual `workflow_dispatch`. In both cases it: + +- Builds the source gem and all native gems (calling the reusable [`build-gems.yml`](../.github/workflows/build-gems.yml) workflow with `--release`). +- Pushes every gem to RubyGems.org using OIDC trusted publishing (no API key required). +- Creates (or updates) a GitHub release for the tag, attaches the `.gem` files and a `CHECKSUMS.txt`, and auto-generates release notes. + +The `push` job is gated on `github.repository == 'ruby/prism'` and runs in the `rubygems.org` GitHub environment, which is the trust anchor for OIDC publishing to RubyGems.org. Pushes that already exist on RubyGems.org are reported as warnings rather than failing the run, so re-runs after a partial failure are safe. + +#### 1. Trigger the release pipeline + +After the preparation commit is on `main`, tag and push: + +```sh +git tag "v${PRISM_VERSION}" +git push origin "v${PRISM_VERSION}" +``` + +The `v*` tag push triggers `publish-gem.yml` automatically. + +If you need to re-trigger the workflow against an existing tag — for example, if the tag-push run failed partway through and you've fixed the cause — go to the Actions tab and use "Run workflow" on `Publish gem to rubygems.org`, passing the tag (e.g. `v1.10.0`) as the `version_tag` input. The workflow will check out that tag, rebuild, and push. + +#### 2. Edit the release notes + +The workflow creates the GitHub release with auto-generated notes. Edit it to paste in the relevant section of `CHANGELOG.md`. + +### Manually + +If for whatever reason (like an emergency), you need to release a version and the Github Actions pipelines aren't available, here's how you can build the full set of native gems. + +#### 1. Build gems + +```sh +bin/build-gems +``` + +This script will: +- Run `bundle update` and `bundle package` to vendor dependencies +- Run a safety check (`compile` and `test`) +- Build all gems via `rake gem:all` (source gem + native gems for all platforms using rake-compiler-dock) +- Place all gems in `./gems/` +- Verify all built gems with `test/prism/packaging/test-gem-file-contents` +- Print SHA256 checksums for inclusion in release notes + +#### 2. Push gems + +Push the native gems first, then the source gem last (so that when users see the new version, native gems are already available): + +```sh +# push native gems first +for gem in gems/prism-*-*.gem ; do + gem push "$gem" +done + +# push source gem last +gem push gems/prism-${PRISM_VERSION}.gem +``` + +#### 3. Push tag + +Push a new tag to the GitHub repository, following the `vX.Y.Z` format: + +```sh +git tag "v${PRISM_VERSION}" +git push origin "v${PRISM_VERSION}" +``` + +#### 4. Create a release + +Update the GitHub release page with a copy of the latest entry in the `CHANGELOG.md` file, including the SHA256 checksums from the build output. diff --git a/lib/prism.rb b/lib/prism.rb index 8f0342724a..9189c449d7 100644 --- a/lib/prism.rb +++ b/lib/prism.rb @@ -136,7 +136,30 @@ def self.find(callable, rubyvm: !!defined?(RubyVM)) # The C extension is the default backend on CRuby. Prism::BACKEND = :CEXT - require "prism/prism" + begin + # The precompiled native libraries are in /lib/prism/ + require_relative "prism/#{RUBY_VERSION[/\d+\.\d+/]}/prism" + rescue LoadError => e + if e.message.include?("GLIBC") + warn(<<~EOM) + + ERROR: It looks like you're trying to use Prism as a precompiled native gem on a system + with an unsupported version of glibc. + + #{e.message} + + If that's the case, then please install Prism via the `ruby` platform gem: + gem install prism --platform=ruby + or, in your Gemfile: + gem "prism", force_ruby_platform: true + + EOM + raise e + end + + # Precompiled library isn't available, fall back to the library compiled at installation time. + require "prism/prism" + end else # The FFI backend is used on other Ruby implementations. Prism::BACKEND = :FFI diff --git a/rakelib/gemspec.rake b/rakelib/gemspec.rake new file mode 100644 index 0000000000..82c71ed9b9 --- /dev/null +++ b/rakelib/gemspec.rake @@ -0,0 +1,22 @@ +namespace :gemspec do + # this task is used by .github/scripts/gem-build to test packaging and installing a gem + desc "Set the gem VERSION to a unique timestamp for testing" + task "fake-version" do + version_re = /spec\.version = "(.*)"/ + + gemspec_path = File.join(__dir__, "..", "prism.gemspec") + gemspec_contents = File.read(gemspec_path) + + current_version_string = version_re.match(gemspec_contents)[1] + current_version = Gem::Version.new(current_version_string) + + fake_version = Gem::Version.new(format("%s.test.%s", current_version.bump, Time.now.strftime("%Y.%m%d.%H%M"))) + + unless gemspec_contents.gsub!(version_re, "spec.version = \"#{fake_version}\"") + raise("Could not hack the VERSION constant") + end + + File.write(gemspec_path, gemspec_contents) + puts "NOTE: wrote version as \"#{fake_version}\"" + end +end diff --git a/test/prism/packaging/test-gem-file-contents b/test/prism/packaging/test-gem-file-contents new file mode 100755 index 0000000000..5ec89d6fcc --- /dev/null +++ b/test/prism/packaging/test-gem-file-contents @@ -0,0 +1,215 @@ +#! /usr/bin/env ruby +# +# This is a standalone script intended to run as part of the CI test suite. +# +# It inspects the contents of a gem file -- both the files and the gemspec -- to ensure we're +# packaging what we expect, and that we're not packaging anything we don't expect. +# +require "bundler/inline" + +gemfile do + source "https://rubygems.org" + gem "minitest" + gem "minitest-reporters" +end + +require "yaml" + +def usage_and_exit(message = nil) + puts "ERROR: #{message}" if message + puts "USAGE: #{File.basename(__FILE__)} [options]" + exit(1) +end + +usage_and_exit if ARGV.include?("-h") +usage_and_exit unless (gemfile = ARGV[0]) +usage_and_exit("#{gemfile} does not exist") unless File.file?(gemfile) +usage_and_exit("#{gemfile} is not a gem") unless /\.gem$/.match?(gemfile) +gemfile = File.expand_path(gemfile) + +gemfile_contents = Dir.mktmpdir do |dir| + Dir.chdir(dir) do + unless system("tar", "-xf", gemfile, "data.tar.gz") + raise "could not unpack gem #{gemfile}" + end + + `tar -ztf data.tar.gz`.split("\n") + end +end + +gemspec = Dir.mktmpdir do |dir| + Dir.chdir(dir) do + unless system("tar", "-xf", gemfile, "metadata.gz") + raise "could not unpack gem #{gemfile}" + end + + YAML.unsafe_load(`gunzip -c metadata.gz`) + end +end + +if ARGV.include?("-v") + puts "---------- gemfile contents ----------" + puts gemfile_contents + puts + puts "---------- gemspec ----------" + puts gemspec.to_ruby + puts +end + +require "minitest/autorun" +require "minitest/reporters" +Minitest::Reporters.use!([Minitest::Reporters::SpecReporter.new]) + +puts "Testing '#{gemfile}' (#{gemspec.platform})" +describe File.basename(gemfile) do + let(:native_ruby_versions) { ["3.3", "3.4", "4.0"] } + + describe "setup" do + it "gemfile contains some files" do + actual = gemfile_contents.length + assert_operator(actual, :>, 10, "expected gemfile to contain more than #{actual} files") + end + + it "gemspec is a Gem::Specification" do + assert_equal(Gem::Specification, gemspec.class) + end + end + + describe "all platforms" do + ["lib"].each do |dir| + it "contains every ruby file in #{dir}/" do + committed_files = `git ls-files #{dir}`.split("\n").grep(/\.rb$/) + generated_files = `git ls-files templates/lib/prism/*.rb.erb`.split("\n").map { |f| f.delete_prefix("templates/").delete_suffix(".erb") } + expected_files = Set.new(committed_files + generated_files) + + skip "looks like this isn't a git repository" if expected_files.empty? + + actual_files = Set.new(gemfile_contents.select { |f| f.start_with?("#{dir}/") }.grep(/\.rb$/)) + assert_equal(expected_files, actual_files) + end + end + + ["test"].each do |dir| + it "does not contain files from #{dir}/" do + actual = gemfile_contents.select { |f| f.start_with?("#{dir}/") }.grep(/\.rb$/) + assert_empty(actual) + end + end + + it "does not contain the Gemfile" do + refute_includes(gemfile_contents, "Gemfile") + end + end + + describe "ruby platform" do + before { skip unless gemspec.platform == Gem::Platform::RUBY } + + it "contains C source files in ext/" do + assert_operator(gemfile_contents.count { |f| File.fnmatch?("ext/**/*.c", f) }, :>=, 2) + end + + it "contains header files" do + assert_operator(gemfile_contents.count { |f| File.fnmatch?("ext/**/*.h", f) }, :>=, 1) + assert_operator(gemfile_contents.count { |f| File.fnmatch?("include/**/*.h", f) }, :>=, 20) + end + + it "contains C source files in src/" do + assert_operator(gemfile_contents.count { |f| File.fnmatch?("src/*.c", f) }, :>=, 10) + end + + it "contains extconf.rb" do + assert_includes(gemfile_contents, "ext/prism/extconf.rb") + end + + it "has extensions set" do + assert_includes(gemspec.extensions, "ext/prism/extconf.rb") + end + + it "sets required_ruby_version appropriately" do + assert( + gemspec.required_ruby_version.satisfied_by?(Gem::Version.new("2.7")), + "required_ruby_version='#{gemspec.required_ruby_version}' should support ruby 2.7" + ) + native_ruby_versions.each do |v| + assert( + gemspec.required_ruby_version.satisfied_by?(Gem::Version.new(v)), + "required_ruby_version='#{gemspec.required_ruby_version}' should support ruby #{v}" + ) + end + end + end + + describe "native platform" do + before { skip unless gemspec.platform.is_a?(Gem::Platform) && gemspec.platform.cpu } + + # aarch64-mingw-ucrt only has cross-rubies for 3.4+, all other platforms have all versions + let(:expected_ruby_versions) do + if gemspec.platform.to_s == "aarch64-mingw-ucrt" + native_ruby_versions.reject { |v| v == "3.3" } + else + native_ruby_versions + end + end + + it "contains expected shared library files" do + expected_ruby_versions.each do |version| + actual = gemfile_contents.find do |p| + File.fnmatch?("lib/prism/#{version}/prism.{so,bundle}", p, File::FNM_EXTGLOB) + end + assert(actual, "expected to find shared library file for ruby #{version}") + end + + actual = gemfile_contents.find do |p| + File.fnmatch?("lib/prism/prism.{so,bundle}", p, File::FNM_EXTGLOB) + end + refute(actual, "did not expect to find shared library file in lib/prism") + + actual = gemfile_contents.find_all do |p| + File.fnmatch?("lib/prism/**/*.{so,bundle}", p, File::FNM_EXTGLOB) + end + assert_equal( + expected_ruby_versions.length, + actual.length, + "expected exactly #{expected_ruby_versions.length} shared library files, found: #{actual.inspect}" + ) + end + + it "has extensions cleared" do + assert_empty(gemspec.extensions) + end + + it "sets required_ruby_version appropriately" do + expected_ruby_versions.each do |v| + assert( + gemspec.required_ruby_version.satisfied_by?(Gem::Version.new(v)), + "required_ruby_version='#{gemspec.required_ruby_version}' should support ruby #{v}" + ) + end + + # verify unsupported versions are excluded + unsupported_ruby_versions = native_ruby_versions - expected_ruby_versions + unsupported_ruby_versions.each do |v| + refute( + gemspec.required_ruby_version.satisfied_by?(Gem::Version.new(v)), + "required_ruby_version='#{gemspec.required_ruby_version}' should NOT support ruby #{v}" + ) + end + + refute( + gemspec.required_ruby_version.satisfied_by?(Gem::Version.new("2.7")), + "required_ruby_version='#{gemspec.required_ruby_version}' should NOT support ruby 2.7" + ) + + refute( + gemspec.required_ruby_version.satisfied_by?(Gem::Version.new("3.2")), + "required_ruby_version='#{gemspec.required_ruby_version}' should NOT support ruby 3.2" + ) + + # verify the upper bound is set (required_ruby_version should not be open-ended) + refute( + gemspec.required_ruby_version.satisfied_by?(Gem::Version.new("9.9")), + "required_ruby_version='#{gemspec.required_ruby_version}' should have an upper bound" + ) + end + end +end