diff --git a/.gitattributes b/.gitattributes index 66bb25bd092d65..1cf55221f145b4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ +*.bat text eol=crlf *.py diff=python +*.py text eol=lf lib/spack/spack/vendor/* linguist-vendored -*.bat text eol=crlf diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 136f798711c372..9d8ebd45a6f751 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,14 +1,15 @@ version: 2 updates: - # Maintain dependencies for GitHub Actions - package-ecosystem: "github-actions" directory: "/" schedule: interval: "daily" - # Requirements to run style checks and build documentation + - package-ecosystem: "pip" directories: - - "/.github/workflows/requirements/style/*" + - "/.github/workflows/requirements/coverage" + - "/.github/workflows/requirements/style" + - "/.github/workflows/requirements/unit_tests" - "/lib/spack/docs" schedule: interval: "daily" diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml index a4952804d01298..e76a3dec4771ca 100644 --- a/.github/workflows/bootstrap.yml +++ b/.github/workflows/bootstrap.yml @@ -38,15 +38,16 @@ jobs: make patch unzip which xz python3 python3-devel tree \ cmake bison - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: Bootstrap clingo run: | . share/spack/setup-env.sh + spack config add config:installer:new spack bootstrap disable github-actions-v2 spack bootstrap disable github-actions-v0.6 - spack -d solve zlib + spack solve zlib tree ~/.spack/bootstrap/store/ clingo-sources: @@ -60,19 +61,20 @@ jobs: if: ${{ matrix.runner != 'ubuntu-latest' }} run: brew install bison tree - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.12" - name: Bootstrap clingo run: | . share/spack/setup-env.sh + spack config add config:installer:new spack bootstrap disable github-actions-v2 spack bootstrap disable github-actions-v0.6 export PATH="$(brew --prefix bison)/bin:$(brew --prefix cmake)/bin:$PATH" - spack -d solve zlib + spack solve zlib tree ~/.spack/bootstrap/store/ gnupg-sources: @@ -91,16 +93,17 @@ jobs: sudo rm $(command -v gpg gpg2 patchelf) done - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: Bootstrap GnuPG run: | . share/spack/setup-env.sh + spack config add config:installer:new spack solve zlib spack bootstrap disable github-actions-v2 spack bootstrap disable github-actions-v0.6 - spack -d gpg list + spack gpg list tree ~/.spack/bootstrap/store/ from-binaries: @@ -119,10 +122,10 @@ jobs: sudo rm $(command -v gpg gpg2 patchelf) done - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: | 3.8 @@ -153,7 +156,8 @@ jobs: - name: Bootstrap GnuPG run: | . share/spack/setup-env.sh - spack -d gpg list + spack config add config:installer:new + spack gpg list tree ~/.spack/bootstrap/store/ windows: @@ -161,10 +165,10 @@ jobs: runs-on: "windows-latest" steps: - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.12" - name: Setup Windows @@ -185,3 +189,31 @@ jobs: spack -d gpg list ./share/spack/qa/validate_last_exit.ps1 tree $env:userprofile/.spack/bootstrap/store/ + + dev-bootstrap: + runs-on: ubuntu-latest + container: registry.access.redhat.com/ubi8/ubi + steps: + - name: Install dependencies + run: | + dnf install -y \ + bzip2 curl gcc-c++ gcc gcc-gfortran git gnupg2 gzip \ + make patch python3.11 tcl unzip which xz + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Setup repo and non-root user + run: | + git --version + git config --global --add safe.directory '*' + git fetch --unshallow + . .github/workflows/bin/setup_git.sh + - name: Setup a virtual environment with platform-python + run: | + python3.11 -m venv ~/platform-spack-311 + source ~/platform-spack-311/bin/activate + pip install --upgrade pip clingo + - name: Bootstrap Spack development environment + run: | + source ~/platform-spack-311/bin/activate + source share/spack/setup-env.sh + spack debug report + spack -d bootstrap now --dev diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index a9728408e414e9..8ba6287783114c 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -55,7 +55,7 @@ jobs: if: github.repository == 'spack/spack' steps: - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Determine latest release tag id: latest @@ -63,7 +63,7 @@ jobs: git fetch --quiet --tags echo "tag=$(git tag --list --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | head -n 1)" | tee -a $GITHUB_OUTPUT - - uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 + - uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 id: docker_meta with: images: | @@ -94,19 +94,19 @@ jobs: fi - name: Upload Dockerfile - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: dockerfiles_${{ matrix.dockerfile[0] }} path: dockerfiles - name: Set up QEMU - uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf + uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Log in to GitHub Container Registry - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -114,13 +114,13 @@ jobs: - name: Log in to DockerHub if: github.event_name != 'pull_request' - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build & Deploy ${{ matrix.dockerfile[0] }} - uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 with: context: dockerfiles/${{ matrix.dockerfile[0] }} platforms: ${{ matrix.dockerfile[1] }} @@ -133,7 +133,7 @@ jobs: needs: deploy-images steps: - name: Merge Artifacts - uses: actions/upload-artifact/merge@6f51ac03b9356f520e9adb1b1b7802705f340c2b + uses: actions/upload-artifact/merge@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: dockerfiles pattern: dockerfiles_* diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a59c628e19d424..2fd6c466ce814d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -25,12 +25,12 @@ jobs: core: ${{ steps.filter.outputs.core }} packages: ${{ steps.filter.outputs.packages }} steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 if: ${{ github.event_name == 'push' || github.event_name == 'merge_group' }} with: fetch-depth: 0 # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 id: filter with: # For merge group events, compare against the target branch (main) @@ -88,12 +88,10 @@ jobs: steps: - name: Success run: | - if [ "${{ needs.prechecks.result }}" == "failure" ] || [ "${{ needs.prechecks.result }}" == "canceled" ]; then - echo "Unit tests failed." - exit 1 - else - exit 0 - fi + [ "${{ needs.prechecks.result }}" = "success" ] && exit 0 + [ "${{ needs.prechecks.result }}" = "skipped" ] && exit 0 + echo "Unit tests failed." + exit 1 coverage: needs: [ unit-tests, prechecks ] @@ -109,12 +107,14 @@ jobs: steps: - name: Status summary run: | - if [ "${{ needs.unit-tests.result }}" == "failure" ] || [ "${{ needs.unit-tests.result }}" == "canceled" ]; then + if [ "${{ needs.unit-tests.result }}" = "success" ] || [ "${{ needs.unit-tests.result }}" = "skipped" ]; then + if [ "${{ needs.bootstrap.result }}" = "success" ] || [ "${{ needs.bootstrap.result }}" = "skipped" ]; then + exit 0 + else + echo "Bootstrap tests failed." + exit 1 + fi + else echo "Unit tests failed." exit 1 - elif [ "${{ needs.bootstrap.result }}" == "failure" ] || [ "${{ needs.bootstrap.result }}" == "canceled" ]; then - echo "Bootstrap tests failed." - exit 1 - else - exit 0 fi diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index a8d0f6dbc5de5e..2d99a07ead3601 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -8,8 +8,8 @@ jobs: upload: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.14' @@ -17,7 +17,7 @@ jobs: run: pip install -r .github/workflows/requirements/coverage/requirements.txt - name: Download coverage artifact files - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: coverage-* path: coverage @@ -28,7 +28,7 @@ jobs: - run: coverage xml - name: "Upload coverage report to CodeCov" - uses: codecov/codecov-action@1e68e06f1dbfde0e4cefc87efeba9e4643565303 + uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 with: verbose: true fail_ci_if_error: false diff --git a/.github/workflows/import-check.yaml b/.github/workflows/import-check.yaml index d282b7ea88133f..641a26dca88da6 100644 --- a/.github/workflows/import-check.yaml +++ b/.github/workflows/import-check.yaml @@ -9,22 +9,22 @@ jobs: continue-on-error: true runs-on: ubuntu-latest steps: - - uses: julia-actions/setup-julia@v2 + - uses: julia-actions/setup-julia@fa02766e078afaaf09b14210362cee14137e6a32 # v3.0.2 with: version: '1.10' - - uses: julia-actions/cache@v2 + - uses: julia-actions/cache@a45e8fa8be21c18a06b7177052533149e61e9b38 # v3.1.0 # PR: use the base of the PR as the old commit - name: Checkout PR base commit if: github.event_name == 'pull_request' - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ github.event.pull_request.base.sha }} path: old # not a PR: use the previous commit as the old commit - name: Checkout previous commit if: github.event_name != 'pull_request' - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 2 path: old @@ -33,11 +33,11 @@ jobs: run: git -C old reset --hard HEAD^ - name: Checkout new commit - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: new - name: Install circular import checker - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: haampie/circular-import-fighter ref: f1c56367833f3c82f6a85dc58595b2cd7995ad48 diff --git a/.github/workflows/prechecks.yml b/.github/workflows/prechecks.yml index 66be75742c0e3c..2ea5c493ae4c48 100644 --- a/.github/workflows/prechecks.yml +++ b/.github/workflows/prechecks.yml @@ -31,10 +31,10 @@ jobs: validate: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Revert spack-stack modifications run: ./REMOVE_SPACK_STACK_MODS_FOR_CI.sh - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.13' - name: Install Python Packages @@ -60,12 +60,12 @@ jobs: style: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 2 - name: Revert spack-stack modifications run: ./REMOVE_SPACK_STACK_MODS_FOR_CI.sh - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.13' - name: Install Python packages diff --git a/.github/workflows/requirements/coverage/requirements.txt b/.github/workflows/requirements/coverage/requirements.txt index 3ee65e02e420a8..ede5d290b580ff 100644 --- a/.github/workflows/requirements/coverage/requirements.txt +++ b/.github/workflows/requirements/coverage/requirements.txt @@ -1 +1 @@ -coverage==7.11.0 +coverage==7.14.0 diff --git a/.github/workflows/requirements/style/requirements.txt b/.github/workflows/requirements/style/requirements.txt index afd9066accadf0..ebf80301f57416 100644 --- a/.github/workflows/requirements/style/requirements.txt +++ b/.github/workflows/requirements/style/requirements.txt @@ -2,10 +2,10 @@ black==25.12.0 clingo==5.8.0 flake8==7.3.0 isort==7.0.0 -mypy==1.19.1 -types-six==1.17.0.20251009 +mypy==1.20.1 vermin==1.8.0 -pylint==4.0.4 +pylint==4.0.5 docutils==0.22.4 ruamel.yaml==0.19.1 slotscheck==0.19.1 +ruff==0.15.11 diff --git a/.github/workflows/requirements/unit_tests/requirements.txt b/.github/workflows/requirements/unit_tests/requirements.txt new file mode 100644 index 00000000000000..99f6f4ba7d974d --- /dev/null +++ b/.github/workflows/requirements/unit_tests/requirements.txt @@ -0,0 +1,5 @@ +pytest==9.0.3 +pytest-cov==7.1.0 +pytest-xdist==3.8.0 +coverage[toml]<=7.11.0 +clingo==5.8.0 diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index 8a5c93bcb35344..c37c363829e010 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -13,7 +13,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 + - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 with: # Issues configuration stale-issue-message: > diff --git a/.github/workflows/triage.yml b/.github/workflows/triage.yml index 9fbc25c8e62069..66611cb8cdc1b2 100644 --- a/.github/workflows/triage.yml +++ b/.github/workflows/triage.yml @@ -19,4 +19,4 @@ jobs: pull-requests: write issues: write steps: - - uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b + - uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1 diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 4ff117dd6699b8..aaa575e444130a 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -49,12 +49,12 @@ jobs: on_develop: false steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: Revert spack-stack modifications run: ./REMOVE_SPACK_STACK_MODS_FOR_CI.sh - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ matrix.python-version }} - name: Install System packages @@ -74,7 +74,7 @@ jobs: run: | # See https://github.com/coveragepy/coveragepy/issues/2082 pip install --upgrade pip pytest pytest-xdist pytest-cov "coverage<=7.11.0" - pip install --upgrade flake8 "isort>=4.3.5" "mypy>=0.900" "click" "black" + pip install --upgrade "mypy>=0.900" "click" "ruff" - name: Setup git configuration run: | # Need this for the git tests to succeed. @@ -98,7 +98,7 @@ jobs: UNIT_TEST_COVERAGE: ${{ matrix.python-version == '3.14' }} run: | share/spack/qa/run-unit-tests - - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: coverage-${{ matrix.os }}-python${{ matrix.python-version }} path: coverage @@ -107,12 +107,12 @@ jobs: shell: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: Revert spack-stack modifications run: ./REMOVE_SPACK_STACK_MODS_FOR_CI.sh - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' - name: Install System packages @@ -128,7 +128,7 @@ jobs: run: "brew install kcov" - name: Install Python packages run: | - pip install --upgrade pip pytest coverage[toml] pytest-xdist + pip install --upgrade pip -r .github/workflows/requirements/unit_tests/requirements.txt - name: Setup git configuration run: | # Need this for the git tests to succeed. @@ -139,7 +139,7 @@ jobs: COVERAGE: true run: | share/spack/qa/run-shell-tests - - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: coverage-shell path: coverage @@ -156,7 +156,7 @@ jobs: dnf install -y \ bzip2 curl gcc-c++ gcc gcc-gfortran git gnupg2 gzip \ make patch tcl unzip which xz - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Revert spack-stack modifications run: ./REMOVE_SPACK_STACK_MODS_FOR_CI.sh - name: Setup repo and non-root user @@ -167,7 +167,7 @@ jobs: . .github/workflows/bin/setup_git.sh - name: Setup a virtual environment with platform-python run: | - /usr/libexec/platform-python -m venv ~/platform-spack + /usr/libexec/platform-python -m venv ~/platform-spack source ~/platform-spack/bin/activate pip install --upgrade pip pytest coverage[toml] pytest-xdist - name: Bootstrap Spack development environment and run unit tests @@ -175,18 +175,18 @@ jobs: source ~/platform-spack/bin/activate source share/spack/setup-env.sh spack debug report - spack -d bootstrap now --dev + spack -d bootstrap now pytest --verbose -x -n3 --dist loadfile -k 'not cvs and not svn and not hg' # Test for the clingo based solver (using clingo-cffi) clingo-cffi: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: Revert spack-stack modifications run: ./REMOVE_SPACK_STACK_MODS_FOR_CI.sh - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.13' - name: Install System packages @@ -195,8 +195,8 @@ jobs: sudo apt-get -y install coreutils gfortran graphviz gnupg2 - name: Install Python packages run: | - pip install --upgrade pip pytest coverage[toml] pytest-cov clingo pytest-xdist - pip install --upgrade flake8 "isort>=4.3.5" "mypy>=0.900" "click" "black" + pip install --upgrade pip -r .github/workflows/requirements/unit_tests/requirements.txt + pip install --upgrade -r .github/workflows/requirements/style/requirements.txt - name: Run unit tests (full suite with coverage) env: COVERAGE: true @@ -209,7 +209,7 @@ jobs: spack bootstrap status spack solve zlib pytest --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml -x -n3 lib/spack/spack/test/concretization/core.py - - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: coverage-clingo-cffi path: coverage @@ -222,19 +222,19 @@ jobs: os: [macos-15-intel, macos-latest] python-version: ["3.14"] steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: Revert spack-stack modifications run: ./REMOVE_SPACK_STACK_MODS_FOR_CI.sh - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ matrix.python-version }} - name: Install Python packages run: | pip install --upgrade pip # See https://github.com/coveragepy/coveragepy/issues/2082 - pip install --upgrade pytest coverage[toml] pytest-xdist pytest-cov "coverage<=7.11.0" + pip install --upgrade -r .github/workflows/requirements/unit_tests/requirements.txt - name: Setup Homebrew packages run: | brew install dash fish gcc gnupg kcov @@ -248,7 +248,7 @@ jobs: spack bootstrap disable spack-install spack solve zlib python3 -m pytest --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml --dist loadfile -x -n4 - - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: coverage-${{ matrix.os }}-python${{ matrix.python-version }} path: coverage @@ -263,17 +263,18 @@ jobs: powershell Invoke-Expression -Command "./share/spack/qa/windows_test_setup.ps1"; {0} runs-on: windows-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: Revert spack-stack modifications run: ./REMOVE_SPACK_STACK_MODS_FOR_CI.sh - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.14' - name: Install Python packages run: | - python -m pip install --upgrade pip pywin32 pytest-cov clingo "coverage<=7.11.0" + python -m pip install --upgrade pip -r .github/workflows/requirements/unit_tests/requirements.txt + python -m pip install --upgrade pip -r .github/workflows/requirements/style/requirements.txt - name: Create local develop run: | ./.github/workflows/bin/setup_git.ps1 @@ -283,7 +284,7 @@ jobs: run: | python -m pytest -x --verbose --cov --cov-config=pyproject.toml ./share/spack/qa/validate_last_exit.ps1 - - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: coverage-windows path: coverage @@ -299,16 +300,16 @@ jobs: steps: - name: Checkout Spack (current) - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: spack-current - name: Checkout Spack (previous) - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: spack-previous ref: ${{ github.event.pull_request.base.sha || github.event.before }} - name: Checkout Spack Packages - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: spack/spack-packages path: spack-packages diff --git a/.gitignore b/.gitignore index 276119d7131bb5..2ea116c2540b19 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ spack-db.* *.in.log *.out.log +CLAUDE.md # Configuration: Ignore everything in /etc/spack, # except defaults and site scopes that ship with spack @@ -19,6 +20,13 @@ spack-db.* !/etc/spack/defaults !/etc/spack/site/README.md +########################### +# Coding agent state +########################### +.claude/ +.gemini/ +.codex/ + ########################### # Python-specific ignores # ########################### diff --git a/.mailmap b/.mailmap index 564a8ad2bc58ce..cb48793427abab 100644 --- a/.mailmap +++ b/.mailmap @@ -29,6 +29,7 @@ Gregory L. Lee Gregory L. Lee Gregory L. Lee Gregory L. Lee Gregory L. Lee Gregory L. Lee Gregory L. Lee Gregory Lee +Harmen Stoppels Harmen Stoppels Ian Lee Ian Lee James Wynne III James Riley Wynne III James Wynne III James Wynne III diff --git a/CHANGELOG.md b/CHANGELOG.md index c18dbee05d75a7..7bd047d1685dcf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,39 @@ +# v1.1.1 (2026-01-14) + +## Usability and performance enhancements + +* solver: do a precheck for non-existing and deprecated versions #51555 +* improvements to solver performance (PRs 51591, 51605, 51612, 51625) +* python 3.14 support (PRs 51686, 51687, 51688, 51689, 51663) +* display when conditions with dependencies in spack info #51588 +* spack repo remove: allow removing from unspecified scope #51563 +* spack compiler info: show non-external compilers too #51718 + +## Improvements to the experimental new installer + +* support forkserver #51788 (for python 3.14 support) +* support --dirty, --keep-stage, and `skip patch` arguments #51558 +* implement --use-buildcache, --cache-only, --use-cache and --only arguments #51593 +* implement overwrite, keep_prefix #51622 +* implement --dont-restage #51623 +* fix logging #51787 + +## Bugfixes + +* repo.py: support rhel 7 #51617 +* solver: match glibc constraints by hash #51559 +* buildache list: list the component prefix not the root #51635 +* solver: fix issue with conditional language dependencies #51692 +* repo.py: fix checking out commits #51695 +* spec parser: ensure toolchains are expanded to different objects #51731 +* RHEL7 git 1.8.3.1 fix #51779 +* RewireTask.complete: return value from \_process\_binary\_cache\_tarball #51825 + +## Documentation + +* docs: fix default projections setting discrepancy #51640 + + # v1.1.0 (2025-11-14) `v1.1.0` features major improvements to **compiler handling** and **configuration management**, a significant refactoring of **externals**, and exciting new **experimental features** like a console UI for parallel installations and concretization caching. @@ -171,6 +207,40 @@ See the [2025.11.0 release](https://github.com/spack/spack-packages/releases/tag/v2025.11.0) of [spack-packages](https://github.com/spack/spack-packages/) for more details. +# v1.0.4 (2026-02-23) + +## Bug fixes + +* Concretizer bugfixes: + * solver: remove a special case for provider weighting #51347 + * solver: improve timeout handling and add Ctrl-C interrupt safety #51341 + * solver: simplify interrupt/timeout logic #51349 +* Repo management bugfixes: + * repo.py: support rhel 7 #51617 + * repo.py: fix checking out commits #51695 + * git: pull_checkout_branch RHEL7 git 1.8.3.1 fix #51779 + * git: fix locking issue in pull_checkout_branch #51854 + * spack repo remove: allow removing from unspecified scope #51563 +* build_environment.py: Prevent deadlock on install process join #51429 +* Fix typo in untrack_env #51554 +* audit.py: fix re.sub(..., N) positional count arg #51735 + +## Enhancements + +* Support Macos Tahoe (#51373, #51394, #51479) +* Support for Python 3.14, except for t-strings (#51686, #51687, #51688, #51697, #51663) +* spack info: show conditional dependencies and licenses; allow filtering #51137 +* Spack fetch less likely to fail due to AI download protections #51496 +* config: relax concurrent_packages to minimum 0 #51840 + * This avoids forward-incompatibility with Spack v1.2 +* Documentation improvements (#51315, #51640) + + +# v1.0.3 (2026-02-20) + +Skipped due to a failure in the release process. + + # v1.0.2 (2025-09-11) ## Bug Fixes diff --git a/CITATION.cff b/CITATION.cff index 59888b51ce29e1..a0d855f11b43ec 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -6,7 +6,7 @@ # Todd Gamblin, Matthew P. LeGendre, Michael R. Collette, Gregory L. Lee, # Adam Moody, Bronis R. de Supinski, and W. Scott Futral. # The Spack Package Manager: Bringing Order to HPC Software Chaos. -# In Supercomputing 2015 (SC’15), Austin, Texas, November 15-20 2015. LLNL-CONF-669890. +# In Supercomputing 2015 (SC'15), Austin, Texas, November 15-20 2015. LLNL-CONF-669890. # # Or, in BibTeX: # @@ -15,10 +15,10 @@ # author = {Gamblin, Todd and LeGendre, Matthew and # Collette, Michael R. and Lee, Gregory L. and # Moody, Adam and de Supinski, Bronis R. and Futral, Scott}, +# booktitle = {Supercomputing 2015 (SC'15)}, # doi = {10.1145/2807591.2807623}, # month = {November 15-20}, # note = {LLNL-CONF-669890}, -# series = {Supercomputing 2015 (SC’15)}, # title = {{The Spack Package Manager: Bringing Order to HPC Software Chaos}}, # url = {https://github.com/spack/spack}, # year = {2015} @@ -61,13 +61,26 @@ preferred-citation: given-names: "Bronis R." - family-names: "Futral" given-names: "Scott" + # collection-title populates `booktitle` in the BibTeX generated by + # GitHub's CFF->BibTeX converter (ruby-cff). Without it, the generated + # @inproceedings entry is missing the required booktitle field. + # See https://github.com/citation-file-format/ruby-cff/issues/79 + collection-title: "Supercomputing 2015 (SC'15)" + collection-type: "proceedings" + publisher: + name: "Association for Computing Machinery" + city: "New York" + region: "NY" + country: "US" conference: - name: "Supercomputing 2015 (SC’15)" + name: "Supercomputing 2015 (SC'15)" city: "Austin" region: "Texas" country: "US" date-start: 2015-11-15 date-end: 2015-11-20 + isbn: "9781450337236" + doi: "10.1145/2807591.2807623" month: 11 year: 2015 identifiers: diff --git a/NEWS.md b/NEWS.md index 0224d3779d3648..89944b5cbbde0a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,6 @@ +## Package API v2.5 +- Added `cuda-lang` and `hip-lang` as language virtuals (analogous to `c`, `cxx`, `fortran`). + ## Package API v2.4 - Added the `%%` sigil to spec syntax, to propagate compiler preferences. diff --git a/bin/haspywin.py b/bin/haspywin.py deleted file mode 100644 index c04285c86ea759..00000000000000 --- a/bin/haspywin.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright Spack Project Developers. See COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) -import subprocess -import sys - - -def getpywin(): - try: - import win32con # noqa: F401 - except ImportError: - print("pyWin32 not installed but is required...\nInstalling via pip:") - subprocess.check_call([sys.executable, "-m", "pip", "-q", "install", "--upgrade", "pip"]) - subprocess.check_call([sys.executable, "-m", "pip", "-q", "install", "pywin32"]) - - -if __name__ == "__main__": - getpywin() diff --git a/bin/spack.bat b/bin/spack.bat index b771fd9789090f..7a7334cf08f0fe 100644 --- a/bin/spack.bat +++ b/bin/spack.bat @@ -12,6 +12,13 @@ :: . /path/to/spack/install/spack_cmd.bat :: @echo off +:: We're directly invoking this script +:: compute spack_root from this +if "%SPACK_ROOT%" =="" ( + pushd %~dp0.. + set SPACK_ROOT=%CD% + popd +) set spack="%SPACK_ROOT%\bin\spack" diff --git a/bin/spack.ps1 b/bin/spack.ps1 index 7cbc8a484d9275..3645081583d0fa 100644 --- a/bin/spack.ps1 +++ b/bin/spack.ps1 @@ -125,9 +125,19 @@ function Invoke-SpackLoad { } } +function Set-SpackRoot { + if ([string]::IsNullOrEmpty($Env:SPACK_ROOT)) { + Push-Location $PSScriptRoot/.. + $Env:SPACK_ROOT = $PWD.Path + Pop-Location + } +} + +Set-SpackRoot $SpackCMD_params, $SpackSubCommand, $SpackSubCommandArgs = Read-SpackArgs $args + if (Compare-CommonArgs $SpackCMD_params) { python "$Env:SPACK_ROOT/bin/spack" $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs exit $LASTEXITCODE diff --git a/etc/spack/defaults/base/config.yaml b/etc/spack/defaults/base/config.yaml index 858eae96275824..65941b96a9108d 100644 --- a/etc/spack/defaults/base/config.yaml +++ b/etc/spack/defaults/base/config.yaml @@ -71,6 +71,9 @@ config: - $user_cache_path/stage # - $spack/var/spack/stage + # Naming format for individual stage directories + stage_name: "spack-stage-{name}-{version}-{hash}" + # Directory in which to run tests and store test results. # Tests will be stored in directories named by date/time and package # name/hash. @@ -89,7 +92,6 @@ config: # This can be purged with `spack clean --misc-cache` misc_cache: $user_cache_path/cache - # Abort downloads after this many seconds if not data is received. # Setting this to 0 will disable the timeout. connect_timeout: 30 @@ -150,31 +152,38 @@ config: # If set to 'urllib', Spack will use python built-in libs to fetch url_fetch_method: urllib - # The maximum number of jobs to use for the build system (e.g. `make`), when - # the -j flag is not given on the command line. Defaults to 16 when not set. - # Note that the maximum number of jobs is limited by the number of cores - # available, taking thread affinity into account when supported. - # For instance: - # - With `build_jobs: 16` and 4 cores available `spack install` will run `make -j4` - # - With `build_jobs: 16` and 32 cores available `spack install` will run `make -j16` - # - With `build_jobs: 2` and 4 cores available `spack install -j6` will run `make -j6` + # The maximum number of jobs to use for the build. When using the old + # installer, this is the number of jobs per package. In the new installer, + # this is the global maximum number of jobs across all packages. When fewer + # cores are available, Spack will use fewer jobs. The `-j` command line + # argument overrides this option. build_jobs: 16 - # The maximum number of concurrent package builds a single Spack instance will run, - # when the `-p` / `--concurrent-packages` flag is not given on the command line. - # Defaults to 1 when not set. - # Generally, big builds like LLVM are going to perform better with a higher -j, and - # builds with lots of dependencies may build better with higher -p. It is currently - # up to the user to balance between -j (build_jobs) and -p (concurrent_packages) - # on `spack install`, but this will be less of an issue in the future when we - # introduce support for the gmake job server and allow more dynamic parallelism. - # This, like `build_jobs`, is also limited by available cores. - # Note: This option has no effect on windows, as parallel builds are disabled on - # windows due to a lack of filesystem locks. - concurrent_packages: 1 - - # Which installer to use: "old" or "new". The new installer is experimental. - installer: old + # The maximum number of concurrent package builds a single Spack process + # will perform. The default value of 0 means no package parallelism when using + # the old installer, and unlimited package parallelism (other than the limit + # set by build_jobs) when using the new installer. Setting this to 1 will + # disable package parallelism in both installers. This option is ignored on + # Windows. + concurrent_packages: 0 + + # Which installer to use: "old" or "new". + installer: new + + # Restrict filesystem and network access during builds. The stage directory, + # install prefix, and system temp directory are always writable. + # Spack-installed dependency prefixes are always readable. Use the allow_read + # and allow_write lists to grant additional permissions to other paths. + # This feature is only available on Linux kernels with Landlock support, and + # only works with the new installer. + sandbox: + enable: false + # Allow TCP network access during the build phase. + allow_network: true + # Additional paths with read and execute permissions. + allow_read: [] + # Additional paths with write and execute permissions. + allow_write: [] # If set to true, Spack will use ccache to cache C compiles. ccache: false diff --git a/etc/spack/defaults/base/packages.yaml b/etc/spack/defaults/base/packages.yaml index 63580f6006531a..42b71d05bae0be 100644 --- a/etc/spack/defaults/base/packages.yaml +++ b/etc/spack/defaults/base/packages.yaml @@ -18,8 +18,9 @@ packages: providers: awk: [gawk] armci: [armcimpi] - blas: [openblas, amdblis] + blas: [openblas] c: [gcc, llvm, intel-oneapi-compilers] + cuda-lang: [cuda] cxx: [gcc, llvm, intel-oneapi-compilers] daal: [intel-oneapi-daal] elf: [elfutils] @@ -32,11 +33,12 @@ packages: glu: [mesa-glu, openglu] golang: [go, gcc] go-or-gccgo-bootstrap: [go-bootstrap, gcc] + hip-lang: [llvm-amdgpu] iconv: [libiconv] ipp: [intel-oneapi-ipp] java: [openjdk, jdk] jpeg: [libjpeg-turbo, libjpeg] - lapack: [openblas, amdlibflame] + lapack: [openblas] libc: [glibc, musl] libgfortran: [gcc-runtime] libglx: [mesa+glx] @@ -108,6 +110,8 @@ packages: buildable: false musl: buildable: false + opengl: + buildable: false spectrum-mpi: buildable: false xl: diff --git a/etc/spack/defaults/windows/config.yaml b/etc/spack/defaults/windows/config.yaml index f54febe957553e..af50575a3c1a60 100644 --- a/etc/spack/defaults/windows/config.yaml +++ b/etc/spack/defaults/windows/config.yaml @@ -3,3 +3,4 @@ config: build_stage:: - '$user_cache_path/stage' stage_name: '{name}-{version}-{hash:7}' + installer: old diff --git a/lib/spack/docs/advanced_topics.rst b/lib/spack/docs/advanced_topics.rst index 1058c99ba03646..61b9cb6188cdbb 100644 --- a/lib/spack/docs/advanced_topics.rst +++ b/lib/spack/docs/advanced_topics.rst @@ -70,6 +70,8 @@ This typically indicates that a package was linked against a system library inst This verification can also be enabled as a post-install hook by setting ``config:shared_linking:missing_library_policy`` to ``error`` or ``warn`` in :ref:`config.yaml `. +.. _filesystem-requirements: + Filesystem Requirements ======================= diff --git a/lib/spack/docs/binary_caches.rst b/lib/spack/docs/binary_caches.rst index 5a0a6c6e03a288..e869b22d0cebe1 100644 --- a/lib/spack/docs/binary_caches.rst +++ b/lib/spack/docs/binary_caches.rst @@ -150,8 +150,8 @@ Build Cache Index Views Build caches can quickly become large and inefficient to search as binaries are added over time. A common work around to this problem is to break the build cache into stacks that target specific applications or workflows. This allows for curation of binaries as smaller collections of packages that push to their own mirrors that each maintain a smaller search area. -However, this approach comes with the tradeoff of requiring much larger storage and computational footprints due to duplication of common dependencies between stacks. -Splitting build caches can also reduce direct fetch hits by reducing the breadth of binaries availabe in a single mirror. +However, this approach comes with the trade off of requiring much larger storage and computational footprints due to duplication of common dependencies between stacks. +Splitting build caches can also reduce direct fetch hits by reducing the breadth of binaries available in a single mirror. To better address the issues with large search areas, build cache index views (or just "views" in this section) were introduced. A view is a named index which provides a curated view into a larger build cache. @@ -167,7 +167,7 @@ View indices are stored similarly to the top level build cache index, but use an Creating a Build Cache Index View """"""""""""""""""""""""""""""""" -Here is an example of creating a view using an active environent. +Here is an example of creating a view using an active environment. .. code-block:: console @@ -188,7 +188,7 @@ If a list of environments is passed while inside of an active environment, the a Updating a Build Cache Index View """"""""""""""""""""""""""""""""" -To prevent accidently overwriting an existing view, it is required to specify how a view should be updated. +To prevent accidentally overwriting an existing view, it is required to specify how a view should be updated. It is possible to use one of two options for updating a view index: ``--force`` or ``--append``. Using the ``--force`` option will replace the index as if the previous one did not exist. The ``--append`` option will first read the existing index, and then add the new specs to it. @@ -332,13 +332,13 @@ Automatic Push to a Build Cache --------------------------------- Sometimes it is convenient to push packages to a build cache immediately after they are installed. -Spack can do this by setting the autopush flag when adding a mirror: +Spack can do this by setting the ``--autopush`` flag when adding a mirror: .. code-block:: console $ spack mirror add --autopush -Or the autopush flag can be set for an existing mirror: +Or the ``--autopush`` flag can be set for an existing mirror: .. code-block:: console diff --git a/lib/spack/docs/bootstrapping.rst b/lib/spack/docs/bootstrapping.rst index dde4cfbe0f56a9..4ef67b5303a807 100644 --- a/lib/spack/docs/bootstrapping.rst +++ b/lib/spack/docs/bootstrapping.rst @@ -47,8 +47,6 @@ Running a command that concretizes a spec, like: .. code-block:: console % spack solve zlib - ==> Bootstrapping clingo from pre-built binaries - ==> Fetching https://mirror.spack.io/bootstrap/github-actions/v0.1/build_cache/darwin-catalina-x86_64/apple-clang-12.0.0/clingo-bootstrap-spack/darwin-catalina-x86_64-apple-clang-12.0.0-clingo-bootstrap-spack-p5on7i4hejl775ezndzfdkhvwra3hatn.spack ==> Installing "clingo-bootstrap@spack%apple-clang@12.0.0~docs~ipo+python build_type=Release arch=darwin-catalina-x86_64" from a buildcache [ ... ] @@ -59,13 +57,7 @@ Users can also bootstrap all Spack's dependencies in a single command, which is .. code-block:: console $ spack bootstrap now - ==> Bootstrapping clingo from pre-built binaries - ==> Fetching https://mirror.spack.io/bootstrap/github-actions/v0.3/build_cache/linux-centos7-x86_64-gcc-10.2.1-clingo-bootstrap-spack-shqedxgvjnhiwdcdrvjhbd73jaevv7wt.spec.json - ==> Fetching https://mirror.spack.io/bootstrap/github-actions/v0.3/build_cache/linux-centos7-x86_64/gcc-10.2.1/clingo-bootstrap-spack/linux-centos7-x86_64-gcc-10.2.1-clingo-bootstrap-spack-shqedxgvjnhiwdcdrvjhbd73jaevv7wt.spack ==> Installing "clingo-bootstrap@spack%gcc@10.2.1~docs~ipo+python+static_libstdcpp build_type=Release arch=linux-centos7-x86_64" from a buildcache - ==> Bootstrapping patchelf from pre-built binaries - ==> Fetching https://mirror.spack.io/bootstrap/github-actions/v0.3/build_cache/linux-centos7-x86_64-gcc-10.2.1-patchelf-0.15.0-htk62k7efo2z22kh6kmhaselru7bfkuc.spec.json - ==> Fetching https://mirror.spack.io/bootstrap/github-actions/v0.3/build_cache/linux-centos7-x86_64/gcc-10.2.1/patchelf-0.15.0/linux-centos7-x86_64-gcc-10.2.1-patchelf-0.15.0-htk62k7efo2z22kh6kmhaselru7bfkuc.spack ==> Installing "patchelf@0.15.0%gcc@10.2.1 ldflags="-static-libstdc++ -static-libgcc" arch=linux-centos7-x86_64" from a buildcache .. _cmd-spack-bootstrap-root: diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index 97724413a500c7..8470c8cd719679 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -104,6 +104,7 @@ "--implicit-namespaces", ".spack/spack-packages/repos/spack_repo", ".spack/spack-packages/repos/spack_repo/builtin/packages", + ".spack/spack-packages/repos/spack_repo/builtin/build_systems/generic.py", ] ) @@ -366,6 +367,7 @@ def setup(sphinx): ("py:class", "spack.traverse.EdgeAndDepth"), ("py:class", "spack.vendor.archspec.cpu.microarchitecture.Microarchitecture"), ("py:class", "spack.vendor.jinja2.Environment"), + ("py:class", "SpecFiltersFactory"), # TypeVar that is not handled correctly ("py:class", "spack.llnl.util.lang.ClassPropertyType"), ("py:class", "spack.llnl.util.lang.K"), @@ -378,6 +380,8 @@ def setup(sphinx): ("py:obj", "spack.llnl.util.lang.KT"), ("py:obj", "spack.llnl.util.lang.V"), ("py:obj", "spack.llnl.util.lang.VT"), + ("py:class", "_P"), + ("py:class", "spack.util.web._R"), ] # The reST default role (used for this markup: `text`) to use for all documents. diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 7ed6ad392e4507..8fe50cd9ded113 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -66,7 +66,7 @@ From lowest to highest precedence: #. **system**: Stored in ``/etc/spack/``. These are settings for this machine or for all machines on which this file system is mounted. - The systm scope overrides the defaults scope. + The system scope overrides the defaults scope. It can be used for settings idiosyncratic to a particular machine, such as the locations of compilers or external packages. Be careful when modifying this scope, as changes here affect all Spack users on a machine. Before putting configuration here, instead consider using the ``site`` scope, which only affects the spack instance it's part of. @@ -85,7 +85,7 @@ From lowest to highest precedence: #. **spack**: Stored in ``$(prefix)/etc/spack/``. Settings here affect only *this instance* of Spack, and they override ``user`` and lower configuration scopes. This is intended for project-specific or single-user spack installations. - This is the the topmost built-in spack scope, and modifying it gives you full control over configuration scopes. + This is the topmost built-in spack scope, and modifying it gives you full control over configuration scopes. For example, it defines the ``user``, ``site``, and ``system`` scopes, so you can use it to remove them completely if you want. #. **environment**: When using Spack :ref:`environments`, Spack reads additional configuration from the environment file. @@ -102,7 +102,7 @@ When configurations conflict, settings from higher-precedence scopes override lo All of these except ``spack`` and ``defaults`` are initially empty, so you don't have to think about the others unless you need them. The most commonly used scopes are ``environment``, ``user``, and ``spack``. -If you forget, you can always see the available configuration scopes in order of precedece wiht the ``spack config scopes`` command:: +If you forget, you can always see the available configuration scopes in order of precedence with the ``spack config scopes`` command:: > spack config scopes -p Scope Path diff --git a/lib/spack/docs/configuring_compilers.rst b/lib/spack/docs/configuring_compilers.rst index 973ec8211029a5..73cf088ae257c4 100644 --- a/lib/spack/docs/configuring_compilers.rst +++ b/lib/spack/docs/configuring_compilers.rst @@ -17,7 +17,7 @@ Compilers can be made available to Spack by: 1. Specifying them as externals in ``packages.yaml``, or 2. Having them installed in the current Spack store, or -3. Having them available as binaries in a buildcache +3. Having them available as binaries in a build cache For convenience, Spack will automatically detect compilers as externals the first time it needs them, if no compiler is available. @@ -36,7 +36,7 @@ You can see which compilers are available to Spack by running ``spack compiler l [e] gcc@10.5.0 [+] gcc@15.1.0 [+] gcc@14.3.0 Compilers marked with an ``[e]`` are system compilers (externals), and those marked with a ``[+]`` have been installed by Spack. -Compilers from remote buildcaches are marked as ``-``, but are not shown by default. +Compilers from remote build caches are marked as ``-``, but are not shown by default. To see them you need a specific option: .. code-block:: console @@ -269,7 +269,7 @@ For example: .. code-block:: spec - $ spack install gcc@14+binutils + $ spack install gcc@14 Once the compiler is installed, you can start using it without additional configuration: @@ -300,4 +300,4 @@ To enable mixing for specific packages, specify an allow-list in the ``compiler_ concretizer: compiler_mixing: ["openssl"] -Adding ``openssl`` to the compiler mixing allow-list does not allow mixing for dependencies of ``openssl``. \ No newline at end of file +Adding ``openssl`` to the compiler mixing allow-list does not allow mixing for dependencies of ``openssl``. diff --git a/lib/spack/docs/containers.rst b/lib/spack/docs/containers.rst index 6086fd8549c2a7..c92acc4d4d8949 100644 --- a/lib/spack/docs/containers.rst +++ b/lib/spack/docs/containers.rst @@ -61,7 +61,7 @@ Exporting Spack installations as Container Images The command .. code-block:: text - + spack buildcache push [--base-image BASE_IMAGE] [--tag TAG] mirror [specs...] creates and pushes a container image to an OCI-compatible container registry, with the ``mirror`` argument specifying a registry (see below). @@ -72,7 +72,7 @@ Container images created this way are **minimal**: they contain only runtime dep Spack itself is *not* included in the resulting image. The arguments are as follows: - + ``--base-image BASE_IMAGE`` Specifies the base image to use for the container. This should be a minimal Linux distribution with a libc that is compatible with the host system. @@ -253,7 +253,7 @@ Since recipes need a little more boilerplate than: RUN spack -e /environment install Spack provides a command to generate customizable recipes for container images. -Customizations include minimizing the size of the image, installing packages in the base image using the system package manager, and setting up a proper entrypoint to run the image. +Customizations include minimizing the size of the image, installing packages in the base image using the system package manager, and setting up a proper entry point to run the image. .. _cmd-spack-containerize: diff --git a/lib/spack/docs/contribution_guide.rst b/lib/spack/docs/contribution_guide.rst index fffacfd001ad9e..c617ac8f6ca38c 100644 --- a/lib/spack/docs/contribution_guide.rst +++ b/lib/spack/docs/contribution_guide.rst @@ -282,7 +282,7 @@ Spack uses GitLab CI for managing the orchestration of build jobs. GitLab Entry Point ~~~~~~~~~~~~~~~~~~ -Add a stack entrypoint to ``share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml``. +Add a stack entry point to ``share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml``. There are two stages required for each new stack: the generation stage and the build stage. The generate stage is defined using the job template ``.generate`` configured with environment variables defining the name of the stack in ``SPACK_CI_STACK_NAME``, the platform (``SPACK_TARGET_PLATFORM``) and architecture (``SPACK_TARGET_ARCH``) configuration, and the tags associated with the class of runners to build on. diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst index 0a7b8bba74751b..a4f0ce070f68e1 100644 --- a/lib/spack/docs/developer_guide.rst +++ b/lib/spack/docs/developer_guide.rst @@ -456,7 +456,7 @@ As the action runs, you should observe output similar to: ssh 5RjFs7LPdtwGG8cwSPkGrdMNg@sfo2.tmate.io https://tmate.io/t/5RjFs7LPdtwGG8cwSPkGrdMNg -The first line is the ssh command neccesary to connect to the server, the second line is a tmate web-ui that also provides access to the ssh server on the runner. +The first line is the ssh command necessary to connect to the server, the second line is a tmate web UI that also provides access to the ssh server on the runner. .. note:: The web UI has occasionally been unresponsive, if it does not respond within ~10s, you'll need to use your local ssh utility. @@ -690,21 +690,78 @@ By running this command before and after the change, you can make sure that your Profiling --------- -Spack has some limited built-in support for profiling, and can report statistics using standard Python timing tools. -To use this feature, supply ``--profile`` to Spack on the command line, before any subcommands. +To profile Spack, use Python's built-in `cProfile `_ module directly: -.. _spack-p: +.. code-block:: console -``spack --profile`` -^^^^^^^^^^^^^^^^^^^ + $ python3 -m cProfile -s cumtime bin/spack find + $ python3 -m cProfile -o profile.out bin/spack find + +.. _debugging-concretization: + +Debugging concretization +------------------------ + +When working on the ASP-based solver in ``lib/spack/spack/solver/``, it is often useful to inspect the raw facts and rules that clingo sees, and to run clingo directly outside of Spack. + +Generating ASP facts +^^^^^^^^^^^^^^^^^^^^ + +The ``spack solve --show=asp`` flag dumps all ASP facts generated for a given spec to stdout: + +.. code-block:: console + + $ spack solve --show=asp zlib-ng > zlib.lp + +The resulting file contains both the package facts (versions, variants, dependencies) and the problem-specific facts derived from the user's configuration. +It can be fed directly to clingo alongside the solver rules. + +Running clingo directly +^^^^^^^^^^^^^^^^^^^^^^^ + +Once you have the facts file, you can invoke clingo directly. +This bypasses Spack's Python layer and lets you iterate on ``.lp`` rule files quickly. -``spack --profile`` output looks like this: +On Linux (includes libc compatibility rules): -.. command-output:: spack --profile graph hdf5 - :ellipsis: 25 +.. code-block:: console + + $ LP_FILES="lib/spack/spack/solver/concretize.lp \ + lib/spack/spack/solver/heuristic.lp \ + lib/spack/spack/solver/display.lp \ + lib/spack/spack/solver/libc_compatibility.lp \ + lib/spack/spack/solver/direct_dependency.lp" + $ clingo --verbose=3 --stats=2 --quiet=1,0,0 [--project-anonymous] \ + --configuration=tweety --opt-strategy=usc,one \ + --heuristic=Domain $LP_FILES zlib.lp + +On macOS, replace ``libc_compatibility.lp`` with ``os_compatibility.lp``. + +Reading the output +^^^^^^^^^^^^^^^^^^ + +``--quiet=1,0,0`` suppresses intermediate models and shows only the optimal answer. +``--stats=2`` appends a detailed statistics block at the end of the output. +The most useful fields are: + +* **Grounding**: total number of ground rules; a sudden increase usually indicates a rule is producing a combinatorial blowup. +* **Solve time**: wall-clock time spent in the search phase alone, excluding grounding. +* **Optimization**: the vector of objective values at each priority level, useful for verifying that the solver is minimizing the right criteria. + +``--verbose=3`` prints each rule as it is grounded, which helps identify which rule is responsible for an unexpected grounding explosion. +Because the output is very large, redirect it to a file and search for the rule body of interest. -The bottom of the output shows the most time-consuming functions, slowest on top. -The profiling support is from Python's built-in tool, `cProfile `_. +If a solve takes a long time to finish, you can interrupt it with ``Ctrl+C``. +The partial statistics printed on interrupt are still useful for diagnosing the bottleneck. + +Running the concretization test suite +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +After modifying any solver ``.lp`` file, verify correctness with: + +.. code-block:: console + + $ pytest -n 8 lib/spack/spack/test/concretization .. _releases: @@ -865,6 +922,9 @@ The majority of the work is to cherry-pick the bug fixes, which ideally should b The backports pull request is always titled ``Backports vX.Y.Z`` and is labelled ``backports``. It is opened from a branch named ``backports/vX.Y.Z`` and targets the ``releases/vX.Y`` branch. +The first commit on the ``backports/vX.Y.Z`` branch should update the Spack version to ``X.Y.Z.dev0``, and should have the commit message ``set version to X.Y.Z.dev0``. +This ensures that if users check out an intermediate commit between two patch releases, Spack reports the version correctly. + Whenever a pull request labelled ``vX.Y.Z`` is merged, cherry-pick the associated squashed commit on ``develop`` to the ``backports/vX.Y.Z`` branch. For pull requests that were rebased (or not squashed), cherry-pick each associated commit individually. Never force-push to the ``backports/vX.Y.Z`` branch. diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst index 9ccba530b01ba7..aaf3e0c69cce81 100644 --- a/lib/spack/docs/environments.rst +++ b/lib/spack/docs/environments.rst @@ -229,10 +229,7 @@ The same rule applies to the ``install`` and ``uninstall`` commands. ==> 0 installed packages $ spack install zlib@1.2.11 - ==> Installing zlib-1.2.11-q6cqrdto4iktfg6qyqcc5u4vmfmwb7iv - ==> No binary for zlib-1.2.11-q6cqrdto4iktfg6qyqcc5u4vmfmwb7iv found: installing from source - ==> zlib: Executing phase: 'install' - [+] ~/spack/opt/spack/linux-rhel7-broadwell/gcc-8.1.0/zlib-1.2.11-q6cqrdto4iktfg6qyqcc5u4vmfmwb7iv + [+] q6cqrdt zlib@1.2.11 ~/spack/opt/spack/linux-rhel7-broadwell/gcc-8.1.0/zlib-1.2.11-q6cqrdto4iktfg6qyqcc5u4vmfmwb7iv (12s) $ spack env activate myenv @@ -242,10 +239,7 @@ The same rule applies to the ``install`` and ``uninstall`` commands. ==> 0 installed packages $ spack install zlib@1.2.8 - ==> Installing zlib-1.2.8-yfc7epf57nsfn2gn4notccaiyxha6z7x - ==> No binary for zlib-1.2.8-yfc7epf57nsfn2gn4notccaiyxha6z7x found: installing from source - ==> zlib: Executing phase: 'install' - [+] ~/spack/opt/spack/linux-rhel7-broadwell/gcc-8.1.0/zlib-1.2.8-yfc7epf57nsfn2gn4notccaiyxha6z7x + [+] yfc7epf zlib@1.2.8 ~/spack/opt/spack/linux-rhel7-broadwell/gcc-8.1.0/zlib-1.2.8-yfc7epf57nsfn2gn4notccaiyxha6z7x (12s) ==> Updating view at ~/spack/var/spack/environments/myenv/.spack-env/view $ spack find @@ -278,16 +272,16 @@ Adding Abstract Specs An abstract spec is the user-specified spec before Spack applies defaults or dependency information. -Users can add abstract specs to an environment using the ``spack add`` command. +You can add abstract specs to an environment using the ``spack add`` command. +This adds the abstract spec as a root of the environment in the ``spack.yaml`` file. The most important component of an environment is a list of abstract specs. -Adding a spec adds it as a root spec of the environment in the user input file (``spack.yaml``). -It does not affect the concrete specs in the lock file (``spack.lock``) and it does not install the spec. +Adding abstract specs does not immediately install anything, nor does it affect the ``spack.lock`` file. +To update the lockfile, the environment must be :ref:`re-concretized `, and to update any installations, the environment must be :ref:`(re)installed `. The ``spack add`` command is environment-aware. It adds the spec to the currently active environment. An error is generated if there isn't an active environment. -All environment-aware commands can also be called using the ``spack -e`` flag to specify the environment. .. code-block:: spec @@ -300,6 +294,10 @@ or $ spack -e myenv add python +.. note:: + + All environment-aware commands can also be called using the ``spack -e`` flag to specify the environment. + .. _cmd-spack-concretize: Concretizing @@ -495,58 +493,59 @@ The ``loads`` file may also be copied out of the environment, renamed, etc. .. _environment_include_concrete: -Included Concrete Environments ------------------------------- +Including Concrete Environments +------------------------------- -Spack environments can create an environment based off of information in already established environments. -You can think of it as a combination of existing environments. -It will gather information from the existing environment's ``spack.lock`` and use that during the creation of this included concrete environment. -When an included concrete environment is created it will generate a ``spack.lock`` file for the newly created environment. +Spack can create an environment that includes information from already concretized environments. +You can think of the new environment as a combination of existing environments. +It uses information from the existing environments' ``spack.lock`` files in the creation of the new environment. +When such an environment is concretized it will generate its own ``spack.lock`` file that contains relevant information from the included environments. -Creating included environments -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Creating combined concrete environments +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ To create a combined concrete environment, you must have at least one existing concrete environment. You will use the command ``spack env create`` with the argument ``--include-concrete`` followed by the name or path of the environment you'd like to include. -Here is an example of how to create a combined environment from the command line. - -.. code-block:: spec +Here is an example of how to create a combined environment from the command line:: $ spack env create myenv $ spack -e myenv add python $ spack -e myenv concretize - $ spack env create --include-concrete myenv included_env + $ spack env create --include-concrete myenv combined_env +You can also include concrete environments directly in the ``spack.yaml`` file. +It involves adding the absolute paths to the concrete environments ``spack.lock`` under the new environment's ``include`` heading. +Spack-specific configuration variables, such as ``$spack``, and environment variables can be used in the include paths as long as the expression expands to an absolute path. +(See :ref:`config-file-variables` for more information.) -You can also include an environment directly in the ``spack.yaml`` file. -It involves adding the ``include_concrete`` heading in the yaml followed by the absolute path to the independent environments. -Note that you may use Spack config variables such as ``$spack`` or environment variables as long as the expression expands to an absolute path. +For example, .. code-block:: yaml spack: + include: + - /absolute/path/to/environment1/spack.lock + - $spack/../path/to/environment2/spack.lock specs: [] concretizer: unify: true - include_concrete: - - /absolute/path/to/environment1 - - $spack/../path/to/environment2 +will include the specs from ``environment1`` and ``environment2`` where the second environment's path is the absolute path of the directory that is relative to the spack root. -Once the ``spack.yaml`` has been updated you must concretize the environment to get the concrete specs from the included environments. +.. note:: -Updating an included environment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -If changes were made to the base environment and you want that reflected in the included environment you will need to re-concretize both the base environment and the included environment for the change to be implemented. -For example: + Once the ``spack.yaml`` file is updated you must concretize the new environment to get the concrete specs from the included environments. + This will produce the combined ``spack.lock`` file. -.. code-block:: spec +Updating a combined environment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +If you want changes made to one of the included environments reflected in the combined environment, then you will need to re-concretize the included environment **then** the combined environment for the change to be incorporated. +For example:: $ spack env create myenv $ spack -e myenv add python $ spack -e myenv concretize - $ spack env create --include-concrete myenv included_env - + $ spack env create --include-concrete myenv combined_env $ spack -e myenv find ==> In environment myenv @@ -555,17 +554,16 @@ For example: ==> 0 installed packages - - $ spack -e included_env find - ==> In environment included_env + $ spack -e combined_env find + ==> In environment combined_env ==> No root specs ==> Included specs python ==> 0 installed packages -Here we see that ``included_env`` has access to the python package through the ``myenv`` environment. -But if we were to add another spec to ``myenv``, ``included_env`` will not be able to access the new information. +Here we see that ``combined_env`` contains the python package from ``myenv`` environment. +But if we were to add another spec to ``myenv``, ``combined_env`` will not know about the other spec. .. code-block:: spec @@ -578,22 +576,21 @@ But if we were to add another spec to ``myenv``, ``included_env`` will not be ab ==> 0 installed packages - - $ spack -e included_env find - ==> In environment included_env + $ spack -e combined_env find + ==> In environment combined_env ==> No root specs ==> Included specs python ==> 0 installed packages -It isn't until you run the ``spack concretize`` command that the combined environment will get the updated information from the re-concretized base environment. +It isn't until you run the ``spack concretize`` command that the combined environment will get the updated information from the re-concretized ``myenv``. .. code-block:: console - $ spack -e included_env concretize - $ spack -e included_env find - ==> In environment included_env + $ spack -e combined_env concretize + $ spack -e combined_env find + ==> In environment combined_env ==> No root specs ==> Included specs perl python @@ -636,7 +633,7 @@ For example, a ``spack.yaml`` manifest file containing some package preference c mpi: [openmpi] # ... -This configuration sets the default mpi provider to be openmpi. +This configuration sets the default ``mpi`` provider to be ``openmpi``. Included configurations ^^^^^^^^^^^^^^^^^^^^^^^ @@ -876,7 +873,7 @@ The valid variables for a ``when`` clause are: The platform string of the default Spack architecture on the system. #. ``os``. - The os string of the default Spack architecture on the system. + The OS string of the default Spack architecture on the system. #. ``target``. The target string of the default Spack architecture on the system. @@ -925,6 +922,115 @@ For example, the following environment has three root packages: ``gcc@8.1.0``, ` This allows for a much-needed reduction in redundancy between packages and constraints. +.. _environment-spec-groups: + +Spec Groups +^^^^^^^^^^^ + +.. versionadded:: 1.2 + +Environments can be organized with named spec groups, enabling you to apply localized configuration overrides and establish concretization dependencies. +This is extremely useful in a couple of common scenarios, as detailed below. + +.. _environment-spec-groups-bootstrapping-compiler: + +Building and using a compiler in a single environment +""""""""""""""""""""""""""""""""""""""""""""""""""""" + +A common use case is to build a recent compiler on top of an existing system and then compile a stack of software with it. +For instance, assume we are interested in building ``hdf5`` and ``libtree`` with ``gcc@15.2``. +The following manifest file would do exactly that: + +.. code-block:: yaml + + spack: + specs: + - group: compiler + specs: + - gcc@15.2 + + - group: apps + needs: [compiler] + specs: + - hdf5 %gcc@15.2 + - libtree %gcc@15.2 + +The ``group:`` attribute allows to name a group of specs, which are then listed under the ``specs:`` attribute in the same object. +The simplest example is the ``compiler`` group composed of just the ``gcc@15.2`` spec. + +To express dependencies among groups of specs the ``needs:`` attribute is used, which is a list of names corresponding to the groups we depend on. +The way this works is that group dependencies are always concretized *before* the current group, and their specs are *always* available for reuse when the current group is concretized. + +.. _environment-spec-groups-configuring-groups: + +Configuring a group of specs +"""""""""""""""""""""""""""" + +Another common scenario is the deployment of different configurations (e.g. CUDA enabled vs. +ROCm enabled) of the same set of software. +As an example, assume we want to install ``gromacs`` and ``quantum-espresso`` for both ``target=x86_64_v3`` and ``target=x86_64_v4``. +That can be done with the following manifest file: + +.. code-block:: yaml + + spack: + - group: apps-x86_64_v3 + specs: + - gromacs + - quantum-espresso + override: + packages: + all: + prefer: + - target=x86_64_v3 + + - group: apps-x86_64_v4 + specs: + - gromacs + - quantum-espresso + override: + packages: + all: + prefer: + - target=x86_64_v4 + +The ``override:`` attribute allows us to override the configuration for a single group of specs. +The overridden part is always added as the *topmost* scope when the current group is concretized. +This ensures the override always takes precedence over other sources of configuration. + +.. _environment-spec-groups-explicit: + +Controlling garbage collection with ``explicit: false`` +""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +By default every spec group is treated as a set of *explicit* roots. +This means its specs are preserved by ``spack gc`` even when nothing else depends on them. +Setting ``explicit: false`` on a group marks its specs as *implicit*, making them eligible for garbage collection once no other installed spec depends on them: + +.. code-block:: yaml + + spack: + specs: + - group: compiler + explicit: false + specs: + - gcc@15.2 + + - group: apps + needs: [compiler] + specs: + - hdf5 %gcc@15.2 + - libtree %gcc@15.2 + +After the apps are installed, ``spack gc`` will remove the compiler once no installed spec has a link or run dependency on it. + +.. note:: + + Flipping ``explicit: false`` on a group that has already been installed does **not** retroactively update the database record for the already-installed specs. + The flag takes effect only for specs installed, or re-installed, after the change. + To immediately mark an existing spec as implicit, use ``spack mark -i ``. + + Modifying Environment Variables ------------------------------- @@ -1027,6 +1133,7 @@ The root specs with their (transitive) link and run type dependencies will be pu all: "{name}/{version}-{compiler.name}" link: all link_type: symlink + link_dirs: true The default for the ``select`` and ``exclude`` values is to select everything and exclude nothing. The default projection is the default view projection (``{}``). @@ -1038,6 +1145,13 @@ The ``link`` attribute allows the following values: The ``link_type`` defaults to ``symlink`` but can also take the value of ``hardlink`` or ``copy``. +.. versionadded:: 1.2 + + The ``link_dirs`` option controls whether directories are symlinked. This is the default + behavior in Spack v1.2 and later. This is an optimization that significantly reduces the time + to create views, and reduces the inode usage of the view. It only applies when ``link_type`` + is set to ``symlink``. If you want to link only non-directory files, set ``link_dirs: false``. + .. tip:: The option ``link: run`` can be used to create small environment views for Python packages. @@ -1086,6 +1200,42 @@ Given the example above, the spec ``zlib@1.2.8`` will be linked into ``/my/view/ If the keyword ``all`` does not appear in the projections configuration file, any spec that does not satisfy any entry in the file will be linked into the root of the view as in a single-prefix view. Any entries that appear below the keyword ``all`` in the projections configuration file will not be used, as all specs will use the projection under ``all`` before reaching those entries. +Group of Specs +"""""""""""""" + +Views can also be applied to a selected list of :ref:`spec groups `. +This can be done by specifying the ``group:`` attribute in the view configuration. +For instance, with the following manifest: + +.. code-block:: yaml + + spack: + concretizer: + unify: true + + packages: + all: + require: + - target=x86_64_v4 + + specs: + - group: compiler + specs: + - gcc@15.2 + + - group: apps + needs: [compiler] + specs: + - hdf5~mpi %gcc@15.2 + - libtree %gcc@15.2 + + view: + apps: + root: ./views/apps + group: apps + +The view will only contain entries from the ``apps`` group, and will not include specs from the ``compiler`` group. + Activating environment views ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1222,7 +1372,7 @@ Adding post-install hooks ^^^^^^^^^^^^^^^^^^^^^^^^^ Another advanced use-case of generated ``Makefile``\s is running a post-install command for each package. -These "hooks" could be anything from printing a post-install message, running tests, or pushing just-built binaries to a buildcache. +These "hooks" could be anything from printing a post-install message, running tests, or pushing just-built binaries to a build cache. This can be accomplished through the generated ``[/]SPACK_PACKAGE_IDS`` variable. Assuming we have an active and concrete environment, we generate the associated ``Makefile`` with a prefix ``example``: @@ -1235,7 +1385,7 @@ And we now include it in a different ``Makefile``, in which we create a target ` This target depends on the particular package installation. In this target we automatically have the target-specific ``HASH`` and ``SPEC`` variables at our disposal. They are respectively the spec hash (excluding leading ``/``), and a human-readable spec. -Finally, we have an entry point target ``push`` that will update the buildcache index once every package is pushed. +Finally, we have an entry point target ``push`` that will update the build cache index once every package is pushed. Note how this target uses the generated ``example/SPACK_PACKAGE_IDS`` variable to define its prerequisites. .. code-block:: Makefile diff --git a/lib/spack/docs/frequently_asked_questions.rst b/lib/spack/docs/frequently_asked_questions.rst index ada520fc06c630..8edcdfb6e79751 100644 --- a/lib/spack/docs/frequently_asked_questions.rst +++ b/lib/spack/docs/frequently_asked_questions.rst @@ -151,6 +151,30 @@ You can also be more specific about what compiler to use for a particular langua These input specs can be simplified using :doc:`toolchains_yaml`. See also :ref:`pitfalls-without-toolchains` for common mistakes to avoid. +.. _faq-concretization-errors: + +How do I debug unexpected or failing concretization? +----------------------------------------------------- + +``spack install`` and ``spack concretize`` may fail with a concretization error when the solver cannot find a package configuration that satisfies all constraints. + +Most of the time, the error message is structured and contains information about which requirements could not be met. +It typically identifies the conflicting constraints and the files where they are defined (e.g., a ``packages.yaml`` entry or a ``conflicts()`` directive in a ``package.py``). +If the cause is clear from the error, you can fix the offending entry directly. + +If it is not obvious *why* the solver made a particular decision -- for example, why it chose a specific version or variant -- run :ref:`spack-solve` to see the full optimization breakdown: + +.. code-block:: console + + $ spack solve + +The output shows the optimization criteria and the weights assigned to each choice. +This makes it possible to trace which preference or requirement is driving an unexpected result. +See also :ref:`faq-concretizer-precedence` for an overview of how criteria are prioritized. + +For a deeper investigation of solver internals, see :ref:`debugging-concretization` in the developer guide. + .. rubric:: Footnotes .. [#f1] The exact list of criteria can be retrieved with the :ref:`spack-solve` command. + See :ref:`faq-concretization-errors` for more information. diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index cef44679a75434..9320f4135b4b49 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -89,7 +89,7 @@ If the search was successful, you can now list known compilers, and get an outpu If no compilers were found, you need to either: * Install further prerequisites, see :ref:`verify-spack-prerequisites`, and repeat the search above. -* Register a buildcache that provides a compiler already available as a binary +* Register a build cache that provides a compiler already available as a binary Once a compiler is available, you can proceed installing your first package: @@ -101,54 +101,14 @@ The output of this command should look similar to the following: .. code-block:: text - [+] /usr (external gcc-10.5.0-zmjbkxxgltryn6hxwzan35qxxw4skbgl) - ==> No binary for compiler-wrapper-1.0-lrmjw5qy3pjeynmxlyfkyzktarvnycfx found: installing from source - ==> Installing compiler-wrapper-1.0-lrmjw5qy3pjeynmxlyfkyzktarvnycfx [2/7] - [+] /usr (external glibc-2.31-rawvy4pmq4nwhk6ipqnesomvstwyopxq) - ==> No binary for gcc-runtime-10.5.0-vchaib2njqlk2cud4a2n33tabq526qjj found: installing from source - ==> Using cached archive: /tmp/try/spack/var/spack/cache/_source-cache/archive/c6/c65a9d2b2d4eef67ab5cb0684d706bb9f005bb2be94f53d82683d7055bdb837c - ==> No patches needed for compiler-wrapper - ==> Installing gcc-runtime-10.5.0-vchaib2njqlk2cud4a2n33tabq526qjj [4/7] - ==> compiler-wrapper: Executing phase: 'install' - ==> No patches needed for gcc-runtime - ==> compiler-wrapper: Successfully installed compiler-wrapper-1.0-lrmjw5qy3pjeynmxlyfkyzktarvnycfx - Stage: 0.00s. Install: 0.00s. Post-install: 0.01s. Total: 0.07s - [+] /home/spack/.local/spack/opt/linux-icelake/compiler-wrapper-1.0-lrmjw5qy3pjeynmxlyfkyzktarvnycfx - ==> gcc-runtime: Executing phase: 'install' - ==> gcc-runtime: Successfully installed gcc-runtime-10.5.0-vchaib2njqlk2cud4a2n33tabq526qjj - Stage: 0.00s. Install: 0.04s. Post-install: 0.05s. Total: 0.14s - [+] /home/spack/.local/spack/opt/linux-icelake/gcc-runtime-10.5.0-vchaib2njqlk2cud4a2n33tabq526qjj - ==> No binary for gmake-4.4.1-ifn6em7abtw6ozpog5ezy565vu66gsrm found: installing from source - ==> Installing gmake-4.4.1-ifn6em7abtw6ozpog5ezy565vu66gsrm [5/7] - ==> Using cached archive: /tmp/try/spack/var/spack/cache/_source-cache/archive/dd/dd16fb1d67bfab79a72f5e8390735c49e3e8e70b4945a15ab1f81ddb78658fb3.tar.gz - ==> No patches needed for gmake - ==> gmake: Executing phase: 'install' - ==> gmake: Successfully installed gmake-4.4.1-ifn6em7abtw6ozpog5ezy565vu66gsrm - Stage: 0.05s. Install: 15.91s. Post-install: 0.01s. Total: 16.00s - [+] /home/spack/.local/spack/opt/linux-icelake/gmake-4.4.1-ifn6em7abtw6ozpog5ezy565vu66gsrm - ==> No binary for zlib-ng-2.2.4-j5ddfaq7nyykn2bovorx73gykhjcl5nz found: installing from source - ==> Installing zlib-ng-2.2.4-j5ddfaq7nyykn2bovorx73gykhjcl5nz [6/7] - ==> Using cached archive: /tmp/try/spack/var/spack/cache/_source-cache/archive/a7/a73343c3093e5cdc50d9377997c3815b878fd110bf6511c2c7759f2afb90f5a3.tar.gz - ==> No patches needed for zlib-ng - ==> zlib-ng: Executing phase: 'autoreconf' - ==> zlib-ng: Executing phase: 'configure' - ==> zlib-ng: Executing phase: 'build' - ==> zlib-ng: Executing phase: 'install' - ==> zlib-ng: Successfully installed zlib-ng-2.2.4-j5ddfaq7nyykn2bovorx73gykhjcl5nz - Stage: 0.03s. Autoreconf: 0.00s. Configure: 3.63s. Build: 2.52s. Install: 0.09s. Post-install: 0.02s. Total: 6.49s - [+] /home/spack/.local/spack/opt/linux-icelake/zlib-ng-2.2.4-j5ddfaq7nyykn2bovorx73gykhjcl5nz - ==> No binary for tcl-8.6.12-6vo5hxeqw5plzd6gvzm74wlfz5stnzcv found: installing from source - ==> Installing tcl-8.6.12-6vo5hxeqw5plzd6gvzm74wlfz5stnzcv [7/7] - ==> Fetching https://mirror.spack.io/_source-cache/archive/26/26c995dd0f167e48b11961d891ee555f680c175f7173ff8cb829f4ebcde4c1a6.tar.gz - [100%] 10.35 MB @ 48.5 MB/s - ==> No patches needed for tcl - ==> tcl: Executing phase: 'autoreconf' - ==> tcl: Executing phase: 'configure' - ==> tcl: Executing phase: 'build' - ==> tcl: Executing phase: 'install' - ==> tcl: Successfully installed tcl-8.6.12-6vo5hxeqw5plzd6gvzm74wlfz5stnzcv - Stage: 0.46s. Autoreconf: 0.00s. Configure: 9.25s. Build: 1m 8.71s. Install: 3.32s. Post-install: 0.68s. Total: 1m 22.61s - [+] /home/spack/.local/spack/opt/linux-icelake/tcl-8.6.12-6vo5hxeqw5plzd6gvzm74wlfz5stnzcv + [e] zmjbkxx gcc@10.5.0 /usr (0s) + [e] rawvy4p glibc@2.31 /usr (0s) + [+] 5qfbgng compiler-wrapper@1.0 /home/spack/.local/spack/opt/linux-icelake/compiler-wrapper-1.0-5qfbgngzoqcjfbwrjn2vh75fr3g25c35 (0s) + [+] vchaib2 gcc-runtime@10.5.0 /home/spack/.local/spack/opt/linux-icelake/gcc-runtime-10.5.0-vchaib2njqlk2cud4a2n33tabq526qjj (0s) + [+] vzazvty gmake@4.4.1 /home/spack/.local/spack/opt/linux-icelake/gmake-4.4.1-vzazvtyn5cjdmg3vkkuau35x7hzu7pyl (12s) + [+] soedrhb zlib-ng@2.3.3 /home/spack/.local/spack/opt/linux-icelake/zlib-ng-2.3.3-soedrhbnpeordiixaib6utcple6tpgya (3s) + [+] u6nztpk tcl@8.6.17 /home/spack/.local/spack/opt/linux-icelake/tcl-8.6.17-u6nztpkhzbga4ul665qqhxucxqk3cins (49s) + Congratulations! You just installed your first package with Spack! @@ -160,7 +120,7 @@ Once you have installed ``tcl``, you can immediately use it by starting the ``tc .. code-block:: console - $ /home/spack/.local/spack/opt/linux-icelake/tcl-8.6.12-6vo5hxeqw5plzd6gvzm74wlfz5stnzcv/bin/tclsh + $ /home/spack/.local/spack/opt/linux-icelake/tcl-8.6.17-u6nztpkhzbga4ul665qqhxucxqk3cins/bin/tclsh >% echo "Hello world!" Hello world! diff --git a/lib/spack/docs/include_yaml.rst b/lib/spack/docs/include_yaml.rst index 3761c33f7d4ae8..65b8078c68a381 100644 --- a/lib/spack/docs/include_yaml.rst +++ b/lib/spack/docs/include_yaml.rst @@ -13,6 +13,8 @@ Include Settings (include.yaml) =============================== Spack allows you to include configuration files through ``include.yaml``, or in the ``include:`` section in an environment. +You can specify includes using local paths, remote paths, and ``git`` URLs. +Included paths become configuration scopes in Spack and can even be used to override built-in scopes. Local files ~~~~~~~~~~~ @@ -23,12 +25,14 @@ You can include a single configuration file or an entire configuration *scope* l include: - /path/to/a/required/config.yaml + - $MY_SPECIAL_CONFIG_FILE + - path: $HOME/path/to/my/project/packages.yaml - path: /path/to/$os/$target/config optional: true - path: /path/to/os-specific/config-dir when: os == "ventura" -Included paths may be absolute, relative (to the configuration file), or they can be specified as URLs. +Included paths may be absolute, relative (to the configuration file), specified as URLs, or provided in environment variables (e.g., ``$MY_SPECIAL_CONFIG_FILE``). * ``optional``: Spack will raise an error when an included configuration file does not exist, *unless* it is explicitly made ``optional: true``, like the second path above. * ``when``: Configuration scopes can also be included *conditionally* with ``when``. @@ -40,50 +44,84 @@ The same conditions and variables in :ref:`Spec List References `_ or `GitLab `_). + If the directory containing the ``include.yaml`` file is not writable when the remote file is downloaded, then the destination will be a temporary directory. + + ``git`` repository files ~~~~~~~~~~~~~~~~~~~~~~~~ You can also include configuration files from a ``git`` repository. -The `branch`, `commit`, or `tag` to be checked out is required. +The ``branch``, ``commit``, or ``tag`` to be checked out is required. A list of relative paths in which to find the configuration files is also required. Inclusion of the repository (and its paths) can be optional or conditional. +If you want to control the :ref:`name of the configuration scope `, you can provide a ``name``. For example, suppose we only want to include the ``config.yaml`` and ``packages.yaml`` files from the `spack/spack-configs `_ repository's ``USC/config`` directory when using the ``centos7`` operating system. -We would then configure the ``include.yaml`` file as follows: - -.. code-block:: yaml +And we want the configuration scope name to start ``common``. +We could then configure the include in, for example, the user scope include file (i.e., ``$HOME/.spack/include.yaml`` by default), as follows:: include: - - git: https://github.com/spack/spack-configs + - name: common + git: https://github.com/spack/spack-configs.git branch: main when: os == "centos7" paths: - USC/config/config.yaml - USC/config/packages.yaml -If the condition is satisfied, then the ``main`` branch of the repository will be cloned and the settings for the two files integrated into Spack's configuration. +.. note:: + + The git URL could be specified through an environment variable (e.g., ``$MY_USC_CONFIG_URL``). + +If the condition is satisfied, then the ``main`` branch of the repository will be cloned -- under ``$HOME/.spack/includes`` -- when configuration scopes are initially created. +Once cloned, the settings for the two files under the ``USC/config`` directory will be integrated into Spack's configuration. +In this example, the new scopes and their paths can be seen by running:: + + $ spack config scopes -p + Scope Path + command_line + spack /Users/username/spack/etc/spack/ + user /Users/username/.spack/ + common:USC/config/config.yaml /Users/username/.spack/includes/common/USC/config/config.yaml + common:USC/config/packages.yaml /Users/username/.spack/includes/common/USC/config/packages.yaml + site /Users/username/spack/etc/spack/site/ + system /etc/spack/ + defaults /Users/username/spack/etc/spack/defaults/ + defaults:darwin /Users/username/spack/etc/spack/defaults/darwin/ + defaults:base /Users/username/spack/etc/spack/defaults/base/ + _builtin + +Since there are two unique paths, each results in a separate configuration scope. +If only the ``USC/config`` directory was listed under ``paths``, then there would be only one configuration scope, named ``USC``, and the configuration settings from all of the configuration files within that directory would be integrated. .. versionadded:: 1.1 ``git:``, ``branch:``, ``commit:``, and ``tag:`` attributes. +.. versionadded:: 1.2 + ``name:`` attribute and git environment variable support. + Precedence ~~~~~~~~~~ @@ -112,7 +150,7 @@ Scopes can tell Spack to prefer to edit their included scopes instead, using ``p path: /path/to/scope/we/want-to-write prefer_modify: true -Now, if the including scope is the highest precedence scope and would otherwise be selected automatically by one fo these commands, they will instead prefer to edit ``preferred``. +Now, if the including scope is the highest precedence scope and would otherwise be selected automatically by one of these commands, they will instead prefer to edit ``preferred``. The including scope can still be modified by using the ``--scope`` argument (e.g., ``spack compiler find --scope NAME``). .. warning:: @@ -143,13 +181,15 @@ If not, Spack will instead use the ``path:`` specified in configuration. .. versionadded:: 1.1 The ``path_override_env_var:`` attribute. +.. _named-config-scopes: + Named configuration scopes ~~~~~~~~~~~~~~~~~~~~~~~~~~ -By default, the included scope names are is constructed by appending ``:`` and the included scope's basename to the parent scope name. +By default, the included scope names are constructed by appending ``:`` and the included scope's basename to the parent scope name. For example, Spack's own ``defaults`` scope includes a ``base`` scope and a platform-specific scope:: - > spack config scopes -p + $ spack config scopes -p Scope Path command_line spack /home/username/spack/etc/spack/ @@ -200,8 +240,8 @@ You can see that all three of these scopes are given meaningful names, and all t The ``user`` and ``system`` scopes can also be disabled by setting ``SPACK_DISABLE_LOCAL_CONFIG``. Finally, the ``user`` scope can be overridden with a path in ``SPACK_USER_CONFIG_PATH`` if it is set. -Overriding scopes by name: -^^^^^^^^^^^^^^^^^^^^^^^^^^ +Overriding scopes by name +^^^^^^^^^^^^^^^^^^^^^^^^^ Configuration scopes have unique names. This means that you can use the ``name:`` attribute to *replace* a builtin scope. @@ -230,7 +270,7 @@ The newly included ``user`` scope will *completely* override the builtin ``user` .. warning:: - Using ``name:`` to override the ``defaults`` scope can have *very* unexpected consequences and is not advised. + Overriding the ``defaults`` scope can have **very** unexpected consequences and is not advised. .. versionadded:: 1.1 The ``name:`` attribute. diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 2aaa4c6d0aaba4..97c8a0200d61af 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -54,6 +54,7 @@ If you're new to Spack and want to start using it, see :doc:`getting_started`, o :caption: Basic Usage package_fundamentals + installing configuring_compilers environments_basics frequently_asked_questions diff --git a/lib/spack/docs/installing.rst b/lib/spack/docs/installing.rst new file mode 100644 index 00000000000000..9d7460f2bf21ac --- /dev/null +++ b/lib/spack/docs/installing.rst @@ -0,0 +1,178 @@ +.. + Copyright Spack Project Developers. See COPYRIGHT file for details. + + SPDX-License-Identifier: (Apache-2.0 OR MIT) + +.. meta:: + :description lang=en: + Learn how Spack installs packages: the interactive terminal UI, parallelism + via a POSIX jobserver, multi-process installs, background execution, and + handling build failures. + +.. _installing: + +Installing Packages +=================== + +This page covers the ``spack install`` experience in detail, including the interactive terminal UI (TUI), parallelism, background execution, and handling build failures. + +Before diving in, ensure you are familiar with :doc:`package_fundamentals` for basic usage and spec syntax. + +.. versionadded:: 1.2 + The TUI and POSIX jobserver are new in Spack 1.2 and require a Unix-like platform. + + +Interactive terminal UI +----------------------- + +By default, ``spack install`` shows live progress inline in the terminal. +Completed packages scroll into terminal history, while active builds update dynamically below the progress header. + +Every package in the install plan is shown with its current status: + +.. code-block:: text + + $ spack install -j16 python + [+] abc1234 zlib@1.3.1 /home/user/spack/opt/spack/... (4s) + [+] def5678 pkgconf@2.2.0 /home/user/spack/opt/spack/... (6s) + [+] 9ab0123 ncurses@6.5 /home/user/spack/opt/spack/... (23s) + Progress: 3/7 +/-: 4 jobs /: filter v: logs n/p: next/prev + [/] cde4567 readline@8.2 configure (11s) + [/] fgh8901 openssl@3.4.1 build (18s) + +Status indicators: + +* ``[+]`` finished successfully +* ``[x]`` failed +* ``[/]``, ``[-]``, ``[\]``, ``[|]`` building (rotating spinner) +* ``[e]`` external + +**Log-following mode**: press ``v`` to switch from the overview to a live view of build output. +Press ``v``, ``q``, or ``Esc`` to return to the overview. + +While in log-following mode, press ``n`` / ``p`` to cycle to the next or previous build. +Press ``/``, type a pattern, and press ``Enter`` to jump to a matching build (``Esc`` cancels the filter). + +When a build fails, press ``v`` to see a parsed error summary and the path to the full log. + + +Parallelism +----------- + +Spack controls parallelism at two levels: the number of build jobs shared across all packages (``-j``), and the number of packages building concurrently (``-p``). + +Build-level parallelism (``-j``) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``-j`` flag controls the **total** number of concurrent build jobs via a POSIX jobserver. +All build processes (``make``, ``cmake``, ``ninja``, etc.) share the same jobserver, so ``-j16`` means at most 16 build jobs across *all* packages combined. +This is the primary concurrency knob. + +.. code-block:: console + + $ spack install -j16 python + +Spack creates a POSIX jobserver compatible with GNU Make's jobserver protocol. +Child build systems automatically respect it through ``MAKEFLAGS``, so total CPU usage stays bounded regardless of how many packages are building concurrently. + +.. note:: + + If an external jobserver is already present in ``MAKEFLAGS``, for example when Spack itself is invoked from inside a larger ``make`` build, Spack attaches to the existing jobserver instead of creating its own. + +Package-level parallelism (``-p``) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``-p`` / ``--concurrent-packages`` flag limits how many packages can be in the build queue simultaneously. +By default there is no limit, and packages are started as jobserver tokens become available. + +.. code-block:: console + + $ spack install -j16 -p4 python + +This builds with 16 total make-jobs but never more than 4 packages at once. + +Dynamic adjustment +^^^^^^^^^^^^^^^^^^ + +You can adjust parallelism while a build is running: + +* Press ``+`` to add a job (increases ``-j`` by 1) +* Press ``-`` to remove a job (decreases ``-j`` by 1) + +When reducing parallelism, Spack waits for currently running jobs to finish before the new limit takes effect; it does not kill active processes. +The progress header shows the adjustment in progress, e.g. ``+/-: 4=>2 jobs``, until the actual count reaches the target. + + +Multi-process and multi-node installs +-------------------------------------- + +Multiple ``spack install`` processes can safely run concurrently, whether on the same machine or across multiple nodes in a cluster with a shared filesystem. +Spack coordinates through :ref:`per-prefix filesystem locks `: before building a package, the process acquires an exclusive lock on its install prefix. +If another process already holds the lock, Spack waits rather than building a second copy. +When a process encounters a prefix that was already installed, it simply skips it and moves on to the next install. + +For best results on a cluster, it's recommended to limit per-process package-level parallelism (e.g., ``spack install -p2``) for better load balancing. + + +Non-interactive mode +-------------------- + +When the controlling process is not a tty, such as in CI pipelines, when redirecting output to a file, or when running in the background, Spack skips the TUI and prints simple line-based status updates instead. +Use ``spack install -v`` to also print build output. + +You can also background builds: + +* **Suspend and resume**: press ``Ctrl-Z`` to suspend the install, then ``bg`` to let it continue in the background or ``fg`` to bring it back. + Child builds are paused while suspended, and resumed when continued in the background or foreground. + The TUI is suppressed while backgrounded and restored on ``fg``. +* **Start in the background**: run ``spack install ... &`` to skip the TUI entirely and build in the background from the start. + +.. tip:: + + You don't need a new terminal or SSH session to keep a build running — just suspend it with ``Ctrl-Z`` and ``bg``, then continue working. + + +Handling failures +----------------- + +By default, Spack continues building other packages when one fails (best-effort). +Use ``--fail-fast`` to stop immediately on the first failure. + +.. code-block:: console + + $ spack install --fail-fast python + +Failed builds show ``[x]`` in the overview. +Navigate to a failed build and press ``v`` to see a parsed error summary and the path to the full log. + +See :ref:`spack install ` for the full set of flags related to debugging and controlling build behavior. + + +Build isolation and sandboxing (Linux) +-------------------------------------- + +Spack can run builds in an unprivileged sandbox to restrict filesystem and network access. +This opt-in feature requires Linux 5.13+ with Landlock support (network restrictions require Linux 6.7+). +Sandboxing is meant for build reproducibility and bug containment rather than acting as a strict security boundary, as package recipes still execute outside the sandbox ahead of the build. + +When enabled, the stage directory, install prefix, system temp directory and ``/dev/null`` are implicitly writable. +Spack-installed dependencies (excluding externals) are implicitly readable. +All other paths must be explicitly allowed in configuration: + +.. code-block:: yaml + + config: + sandbox: + enable: true # Enable for all builds + allow_network: false # Disable TCP network access during the build phase + allow_read: # Additional paths with read and execute permissions + - /usr + allow_write: # Additional paths with write and execute permissions + - /scratch + +The sandbox activates immediately after source extraction and prefix creation. +Note that network restrictions only apply during the build phases, leaving Spack's own fetch operations unaffected. + +File system restrictions are complementary to existing file permissions and ACLs; they cannot grant access to files the user does not already have permission to read or write. + +Spack's sandboxing complements external containerization tools like Podman or Bubblewrap: while a container must grant the main Spack process write access to the entire software store, Landlock dynamically confines each build subprocess strictly to its exact, package-specific install prefix. diff --git a/lib/spack/docs/module_file_support.rst b/lib/spack/docs/module_file_support.rst index 86036db649b1c3..dbae34821e2ddf 100644 --- a/lib/spack/docs/module_file_support.rst +++ b/lib/spack/docs/module_file_support.rst @@ -399,7 +399,7 @@ For instance, the following config options, will add a ``python3.12`` to module names of packages compiled with Python 3.12, and similarly for all specs depending on ``python@3``. This is useful to know which version of Python a set of Python extensions is associated with. -Likewise, the ``openblas`` string is attached to any program that has openblas in the spec, most likely via the ``+blas`` variant specification. +Likewise, the ``openblas`` string is attached to any program that has ``openblas`` in the spec, most likely via the ``+blas`` variant specification. The most heavyweight solution to module naming is to change the entire naming convention for module files. This uses the projections format covered in :ref:`view_projections`. @@ -413,7 +413,7 @@ This uses the projections format covered in :ref:`view_projections`. all: "{name}/{version}-{compiler.name}-{compiler.version}-module" ^mpi: "{name}/{version}-{^mpi.name}-{^mpi.version}-{compiler.name}-{compiler.version}-module" -will create module files that are nested in directories by package name, contain the version and compiler name and version, and have the word ``module`` before the hash for all specs that do not depend on mpi, and will have the same information plus the MPI implementation name and version for all packages that depend on mpi. +will create module files that are nested in directories by package name, contain the version and compiler name and version, and have the word ``module`` before the hash for all specs that do not depend on ``mpi``, and will have the same information plus the MPI implementation name and version for all packages that depend on ``mpi``. When specifying module names by projection for Lmod modules, we recommend NOT including names of dependencies (e.g., MPI, compilers) that are already in the Lmod hierarchy. @@ -447,7 +447,7 @@ When specifying module names by projection for Lmod modules, we recommend NOT in :class: note When ``lmod`` is activated Spack will generate a set of hierarchical lua module files that are understood by Lmod. - The hierarchy always contains the ``Core`` and ``Compiler`` layers, but can be extended to include any virtual packages present in Spack. + The hierarchy always contains the ``Core`` and ``Compiler`` layers, but can be extended to include any package or virtual package in Spack. A case that could be useful in practice is for instance: .. code-block:: yaml @@ -460,13 +460,14 @@ When specifying module names by projection for Lmod modules, we recommend NOT in core_compilers: - "gcc@4.8" core_specs: - - "python" + - "r" hierarchy: - "mpi" - "lapack" + - "python" - that will generate a hierarchy in which the ``lapack`` and ``mpi`` layer can be switched independently. - This allows a site to build the same libraries or applications against different implementations of ``mpi`` and ``lapack``, and let Lmod switch safely from one to the other. + that will generate a hierarchy in which the ``python``, ``lapack`` and ``mpi`` layer can be switched independently. + This allows a site to build the same libraries or applications against different implementations of ``mpi`` and ``lapack``, and with different versions of those implementations and of ``python``, and let Lmod switch safely from among the resulting installs. All packages built with a compiler in ``core_compilers`` and all packages that satisfy a spec in ``core_specs`` will be put in the ``Core`` hierarchy of the lua modules. diff --git a/lib/spack/docs/package_api.rst b/lib/spack/docs/package_api.rst index 53fd0b10d6c1bb..287285b43bc20f 100644 --- a/lib/spack/docs/package_api.rst +++ b/lib/spack/docs/package_api.rst @@ -34,6 +34,38 @@ Compatibility between Spack and :doc:`package repositories ` is ma Spack version |spack_version| supports package repositories with a Package API version between |min_package_api_version| and |package_api_version|, inclusive. +Changelog +--------- + +**v2.5** *(Spack v1.2.0)* + +* Added ``cuda-lang`` and ``hip-lang`` as language virtuals, analogous to ``c``, ``cxx``, and ``fortran``. + Packages that use CUDA or HIP can now declare explicit language dependencies on these virtuals. + +**v2.4** *(Spack v1.0.3)* + +* The ``%%`` operator can be used on input specs to set propagated preferences, which is particularly useful for ``unify: false`` environments. + +**v2.3** *(Spack v1.0.3)* + +* The :func:`~spack.package.version` directive now supports the ``git_sparse_paths`` parameter, allowing sparse checkouts when fetching from git repositories. + +**v2.2** *(Spack v1.0.0)* + +* Renamed implicit builder attributes with backward compatibility: + + * ``legacy_buildsystem`` to ``default_buildsystem``, + * ``legacy_methods`` to ``package_methods``, + * ``legacy_attributes`` to ``package_attributes``, + * ``legacy_long_methods`` to ``package_long_methods``. + +* Exported :class:`~spack.package.GenericBuilder`, :class:`~spack.package.Package`, and :class:`~spack.package.BuilderWithDefaults` from :mod:`spack.package`. +* Exported numerous utility functions and classes for file operations, library/header search, macOS/Windows support, compiler detection, and build system helpers. + +**v2.1** *(Spack v1.0.0)* + +* Exported :class:`~spack.package.CompilerError` and :class:`~spack.package.SpackError` from :mod:`spack.package`. + Spack Package API Reference --------------------------- diff --git a/lib/spack/docs/package_fundamentals.rst b/lib/spack/docs/package_fundamentals.rst index 8a6e86db9451b0..48fa1458811fa8 100644 --- a/lib/spack/docs/package_fundamentals.rst +++ b/lib/spack/docs/package_fundamentals.rst @@ -133,24 +133,15 @@ For example, to install the latest version of the ``mpileaks`` package, you migh If ``mpileaks`` depends on other packages, Spack will install the dependencies first. It then fetches the ``mpileaks`` tarball, expands it, verifies that it was downloaded without errors, builds it, and installs it in its own directory under ``$SPACK_ROOT/opt``. -You'll see a number of messages from Spack, a lot of build output, and a message that the package is installed. .. code-block:: spec $ spack install mpileaks ... dependency build output ... - ==> Installing mpileaks-1.0-ph7pbnhl334wuhogmugriohcwempqry2 - ==> No binary for mpileaks-1.0-ph7pbnhl334wuhogmugriohcwempqry2 found: installing from source - ==> mpileaks: Executing phase: 'autoreconf' - ==> mpileaks: Executing phase: 'configure' - ==> mpileaks: Executing phase: 'build' - ==> mpileaks: Executing phase: 'install' - [+] ~/spack/opt/linux-rhel7-broadwell/gcc-8.1.0/mpileaks-1.0-ph7pbnhl334wuhogmugriohcwempqry2 + [+] ph7pbnh mpileaks@1.0 ~/spack/opt/linux-rhel7-broadwell/gcc-8.1.0/mpileaks-1.0-ph7pbnhl334wuhogmugriohcwempqry2 (5s) The last line, with the ``[+]``, indicates where the package is installed. -Add the Spack debug option (one or more times) -- ``spack -d install mpileaks`` -- to get additional (and even more verbose) output. - Building a specific version ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -176,7 +167,7 @@ We'll talk more about how you can use them to customize an installation in :ref: Reusing installed dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -By default, when you run ``spack install``, Spack tries hard to reuse existing installations as dependencies, either from a local store or from remote buildcaches, if configured. +By default, when you run ``spack install``, Spack tries hard to reuse existing installations as dependencies, either from a local store or from remote build caches, if configured. This minimizes unwanted rebuilds of common dependencies, in particular if you update Spack frequently. In case you want the latest versions and configurations to be installed instead, you can add the ``--fresh`` option: @@ -539,7 +530,7 @@ If you want to find only libelf versions greater than version 0.8.12, you could -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- libelf@0.8.12 libelf@0.8.13 -Finding just the versions of libdwarf built with a particular version of libelf would look like this: +Finding just the versions of ``libdwarf`` built with a particular version of libelf would look like this: .. code-block:: spec @@ -549,7 +540,7 @@ Finding just the versions of libdwarf built with a particular version of libelf libdwarf@20130729-d9b90962 We can also search for packages that have a certain attribute. -For example, ``spack find libdwarf +debug`` will show only installations of libdwarf with the 'debug' compile-time option enabled. +For example, ``spack find libdwarf +debug`` will show only installations of ``libdwarf`` with the 'debug' compile-time option enabled. The full spec syntax is discussed in detail in :ref:`sec-specs`. @@ -656,7 +647,7 @@ You can use this with tools like `jq `_ to quickly create J ^^^^^^^^^^^^^^ It's often the case that you have two versions of a spec that you need to disambiguate. -Let's say that we've installed two variants of zlib, one with and one without the optimize variant: +Let's say that we've installed two variants of ``zlib``, one with and one without the optimize variant: .. code-block:: spec @@ -690,7 +681,7 @@ We run the command and quickly encounter a problem because two versions are inst c) use `spack uninstall --all` to uninstall ALL matching specs. Oh no! -We can see from the above that we have two different versions of zlib installed, and the only difference between the two is the hash. +We can see from the above that we have two different versions of ``zlib`` installed, and the only difference between the two is the hash. This is a good use case for ``spack diff``, which can easily show us the "diff" or set difference between properties for two packages. Let's try it out. Because the only difference we see in the ``spack find`` view is the hash, let's use ``spack diff`` to look for more detail. @@ -721,7 +712,7 @@ Here is an example: Awesome! Now let's read the diff. -It tells us that our first zlib was built with ``~optimize`` (``False``) and the second was built with ``+optimize`` (``True``). +It tells us that our first ``zlib`` was built with ``~optimize`` (``False``) and the second was built with ``+optimize`` (``True``). You can't see it in the docs here, but the output above is also colored based on the content being an addition (+) or subtraction (-). This is a small example, but you will be able to see differences for any attributes on the installation spec. @@ -808,7 +799,7 @@ For example, this will add the ``mpich`` package built with ``gcc`` to your path ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/mpich@3.0.4/bin/mpicc These commands will add appropriate directories to your ``PATH`` and ``MANPATH`` according to the :ref:`prefix inspections ` defined in your modules configuration. -When you no longer want to use a package, you can type unload or unuse similarly: +When you no longer want to use a package, you can type ``spack unload``: .. code-block:: spec diff --git a/lib/spack/docs/package_review_guide.rst b/lib/spack/docs/package_review_guide.rst index a62066b3a5882f..585251bb31cf5a 100644 --- a/lib/spack/docs/package_review_guide.rst +++ b/lib/spack/docs/package_review_guide.rst @@ -129,7 +129,7 @@ This commonly happens with Python packages. For example, the case of one or more letters in the package name may change at some point (e.g., `py-sphinx `_). Also, dashes may be replaced with underscores (e.g., `py-scikit-build `_). In some cases, both changes can occur for the same package. -As these examples illlustrate, it is sometimes possible to add a ``url_for_version`` method to override the default derived URL to ensure the correct one is returned. +As these examples illustrate, it is sometimes possible to add a ``url_for_version`` method to override the default derived URL to ensure the correct one is returned. If older versions are no longer available and there is a chance someone has the package in a build cache, the usual approach is to first suggest :ref:`deprecating ` them in the package. @@ -300,7 +300,7 @@ They only need to be checked in a review when versions are being added or remove Dependencies affected by such changes should be confirmed, when possible, and *at least* when the Contributor is not a Maintainer of the package. **Solutions.** -In some cases, the needed change may be as simple as ensuring the version range and or variant options in the dependency are accurate. +In some cases, the needed change may be as simple as ensuring the version ranges (see :ref:`version_compatibility`) and/or variant options in the dependency are accurate. In others, one or more of the dependencies needed by new versions are missing and need to be added. Or there may be dependencies that are no longer relevant when versions requiring them are removed, meaning the dependencies should be removed as well. diff --git a/lib/spack/docs/packages_yaml.rst b/lib/spack/docs/packages_yaml.rst index 867d18b3793143..d8bce707d971f3 100644 --- a/lib/spack/docs/packages_yaml.rst +++ b/lib/spack/docs/packages_yaml.rst @@ -62,7 +62,7 @@ If Spack is asked to build a package that uses one of these MPIs as a dependency Note that the specified path is the top-level install prefix, not the ``bin`` subdirectory. ``packages.yaml`` can also be used to specify modules to load instead of the installation prefixes. -The following example says that module ``CMake/3.7.2`` provides cmake version 3.7.2. +The following example says that module ``CMake/3.7.2`` provides CMake version 3.7.2. .. code-block:: yaml @@ -155,7 +155,7 @@ Spack can be configured with every MPI provider not buildable individually, but Spack can then use any of the listed external implementations of MPI to satisfy a dependency, and will choose among them depending on the compiler and architecture. -In cases where the concretizer is configured to reuse specs, and other ``mpi`` providers (available via stores or buildcaches) are not desirable, Spack can be configured to require specs matching only the available externals: +In cases where the concretizer is configured to reuse specs, and other ``mpi`` providers (available via stores or build caches) are not desirable, Spack can be configured to require specs matching only the available externals: .. code-block:: yaml @@ -173,7 +173,7 @@ In cases where the concretizer is configured to reuse specs, and other ``mpi`` p - spec: "openmpi@1.4.3+debug" prefix: /opt/openmpi-1.4.3-debug -This configuration prevents any spec using MPI and originating from stores or buildcaches to be reused, unless it matches the requirements under ``packages:mpi:require``. +This configuration prevents any spec using MPI and originating from stores or build caches to be reused, unless it matches the requirements under ``packages:mpi:require``. For more information on requirements see :ref:`package-requirements`. Specifying dependencies among external packages diff --git a/lib/spack/docs/packaging_guide_advanced.rst b/lib/spack/docs/packaging_guide_advanced.rst index 26a2bc322c6222..73d5ca49218d3d 100644 --- a/lib/spack/docs/packaging_guide_advanced.rst +++ b/lib/spack/docs/packaging_guide_advanced.rst @@ -27,99 +27,102 @@ This section of the packaging guide covers a few advanced topics. Multiple build systems ---------------------- -It is not uncommon for a package to use different build systems across different versions or platforms. -For instance, a project might migrate from Autotools to CMake, or use a different build system on Windows than on UNIX. +Packages may use different build systems over time or across platforms. Spack is designed to handle this seamlessly within a single ``package.py`` file. -While Spack uses one package class per recipe, it can manage multiple build systems by associating different *builder* classes with the package. -This design makes supporting multiple build systems straightforward and maintainable. -The following changes are needed to support multiple build systems in a package: +Let's assume we work with ``curl`` and that the package is built using Autotools so far: -1. The package class should derive from *multiple base classes*, such as ``CMakePackage`` and ``AutotoolsPackage``. -2. The ``build_system`` directive is used to declare the available build systems and specify the default one. -3. The :doc:`build instructions ` are specified in *separate builder classes*. +.. code-block:: python + + from spack_repo.builtin.build_systems.autotools import AutotoolsPackage + + + class Curl(AutotoolsPackage): + + depends_on("zlib-api") + + def configure_args(self): + return [f"--with-zlib={self.spec['zlib-api'].prefix}"] + +To add CMake as a further build system we need to: -Here is a simple example of a package that supports both CMake and Autotools: +1. Add another base to the ``Curl`` package class (in our case ``cmake.CMakePackage``), +2. Explicitly declare which build systems are supported using the ``build_system`` directive, +3. Move the :doc:`build instructions ` in *separate builder classes*. .. code-block:: python - from spack.package import * - from spack_repo.builtin.build_systems import cmake, autotools + from spack_repo.builtin.build_systems import autotools, cmake - class Example(cmake.CMakePackage, autotools.AutotoolsPackage): - variant("my_feature", default=True) - build_system("cmake", "autotools", default="cmake") + class Curl(cmake.CMakePackage, autotools.AutotoolsPackage): + build_system("autotools", "cmake", default="cmake") - class CMakeBuilder(cmake.CMakeBuilder): - def cmake_args(self): - return [self.define_from_variant("MY_FEATURE", "my_feature")] + depends_on("zlib-api") class AutotoolsBuilder(autotools.AutotoolsBuilder): def configure_args(self): - return self.with_or_without("my-feature", variant="my_feature") + return [f"--with-zlib={self.spec['zlib-api'].prefix}"] -When defining a package like this, Spack automatically makes the ``build_system`` **variant** available, which can be used to pick the desired build system at install time. -For example -.. code-block:: spec + class CMakeBuilder(cmake.CMakeBuilder): + def cmake_args(self): + return [self.define_from_variant("USE_NGHTTP2", "nghttp2")] - $ spack install example +feature build_system=cmake +In general, with multiple build systems there is a clear split between the :doc:`package metadata ` and the :doc:`build instructions `: -makes Spack pick the ``CMakeBuilder`` class and runs ``cmake -DMY_FEATURE:BOOL=ON``. +1. The directives such as ``depends_on``, ``variant``, ``patch`` go into the package class +2. The build phase functions like ``configure``, ``build`` and ``install``, and helper functions such as ``cmake_args`` or ``configure_args`` go into the builder classes -Similarly +When ``curl`` is concretized, we can select its build system using the ``build_system`` variant, which is available for every package: .. code-block:: spec - $ spack install example +feature build_system=autotools + $ spack install curl build_system=cmake -will pick the ``AutotoolsBuilder`` class and runs ``./configure --with-my-feature``. +Override "phases" of a build system +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -With multiple build systems, we have a clear split between the :doc:`package metadata ` and the :doc:`build instructions `. -The directives such as ``depends_on``, ``variant``, ``patch`` go into the package class, whereas build phase functions like ``configure``, ``build`` and ``install``, and helper functions such as ``cmake_args`` or ``configure_args`` go into the builder classes. +Sometimes package recipes need to override entire :ref:`phases ` of a build system. +Let's assume this happens for ``cp2k``: -.. note:: - - The signature of certain methods changes when moving from a single build system to multiple build systems. +.. code-block:: python - Suppose you add support for CMake in the following Autotools package: + from spack.package import * + from spack_repo.builtin.build_systems import autotools - .. code-block:: python - from spack.package import * - from spack_repo.builtin.build_systems import autotools + class Cp2k(autotools.AutotoolsPackage): + def install(self, spec: Spec, prefix: str) -> None: + # ...existing code... + pass +If we want to add CMake as another build system we need to remember that the signature of phases changes when moving from the ``Package`` to the ``Builder`` class: - class Example(autotools.AutotoolsPackage): - def install(self, spec: Spec, prefix: str) -> None: - # ...existing code... - pass - - Then you should move the install method to the appropriate builder class, and change its signature: +.. code-block:: python - .. code-block:: python + from spack.package import * + from spack_repo.builtin.build_systems import autotools, cmake - from spack.package import * - from spack_repo.builtin.build_systems import autotools, cmake + class Cp2k(autotools.AutotoolsPackage, cmake.CMakePackage): + build_system("autotools", "cmake", default="cmake") - class Example(autotools.AutotoolsPackage, cmake.CMakePackage): - build_system("autotools", "cmake", default="cmake") + class AutotoolsBuilder(autotools.AutotoolsBuilder): + def install(self, pkg: Cp2k, spec: Spec, prefix: str) -> None: + # ...existing code... + pass - class AutotoolsBuilder(autotools.AutotoolsBuilder): - def install(self, pkg: Example, spec: Spec, prefix: str) -> None: - # ...existing code... - pass +The ``install`` method now takes the ``Package`` instance as the first argument, since ``self`` refers to the builder class. - Notice that the install method now takes the package instance as the first argument. - This is because ``self`` refers to the builder class, not the package class. +Add dependencies conditional on a build system +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Build dependencies typically depend on the choice of the build system. -An effective way to handle this is to use a ``with when("build_system=...")`` block to specify dependencies that are only relevant for a specific build system. +Many build dependencies are conditional on which build system is chosen. +An effective way to handle this is to use a ``with when("build_system=...")`` block to specify dependencies that are only relevant for a specific build system: .. code-block:: python @@ -127,7 +130,7 @@ An effective way to handle this is to use a ``with when("build_system=...")`` bl from spack_repo.builtin.build_systems import cmake, autotools - class Example(cmake.CMakePackage, autotools.AutotoolsPackage): + class Cp2k(cmake.CMakePackage, autotools.AutotoolsPackage): build_system("cmake", "autotools", default="cmake") @@ -145,10 +148,10 @@ An effective way to handle this is to use a ``with when("build_system=...")`` bl depends_on("perl", type="build") depends_on("pkgconfig", type="build") -In the previous example, users could pick the desired build system at install time by specifying the ``build_system`` variant. -Much more commonly, packages transition from one build system to another from one version to the next. -That is, a package might use Autotools in version ``0.63`` and CMake in version ``0.64``. -In such cases we have to use the ``build_system`` directive to indicate when which build system can be used: +Transition from one build system to another +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Packages that transition from one build system to another can be modeled using :ref:`conditional variant values `: .. code-block:: python @@ -156,7 +159,7 @@ In such cases we have to use the ``build_system`` directive to indicate when whi from spack_repo.builtin.build_systems import cmake, autotools - class Example(cmake.CMakePackage, autotools.AutotoolsPackage): + class Cp2k(cmake.CMakePackage, autotools.AutotoolsPackage): build_system( conditional("cmake", when="@0.64:"), @@ -166,16 +169,32 @@ In such cases we have to use the ``build_system`` directive to indicate when whi In the example, the directive imposes a change from ``Autotools`` to ``CMake`` going from ``v0.63`` to ``v0.64``. -We have seen how users can run ``spack install example build_system=cmake`` to pick the desired build system. -The same can be done in ``depends_on`` statements, which has certain use cases. -A notable example is when a CMake package *needs* a CMake config file for its dependency, which is only generated when the dependency is built with CMake (and not Autotools). -In that case, you can *force* the choice of the build system of the dependency: +Inherit from a package with multiple build systems +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Customizing a package supporting multiple build systems is straightforward. +If we need to only customize the metadata, we can just define the derived package class. + +For instance, let's assume we want to add a new version to the ``silo`` package: + +.. code-block:: python + + from spack_repo.builtin.packages.silo.package import Silo as BuiltinSilo + + class Silo(BuiltinSilo): + # Version not in builtin.silo + version("special_version") + +If we don't define any builder, Spack will reuse the custom builder from ``builtin.silo`` by default. +If we need to customize the builder too, we just have to inherit from it, like any other Python class: .. code-block:: python - class Dependent(CMakePackage): + from spack_repo.builtin.packages.silo.package import CMakeBuilder as SiloCMakeBuilder - depends_on("example build_system=cmake") + class CMakeBuilder(SiloCMakeBuilder): + def cmake_args(self): + return [self.define_from_variant("USE_NGHTTP2", "nghttp2")] .. _make-package-findable: diff --git a/lib/spack/docs/packaging_guide_build.rst b/lib/spack/docs/packaging_guide_build.rst index b36a3322206c8b..026096ce67b7eb 100644 --- a/lib/spack/docs/packaging_guide_build.rst +++ b/lib/spack/docs/packaging_guide_build.rst @@ -85,7 +85,7 @@ From simplest to most complex, the following are the most common ways to customi For example, for ``AutotoolsPackage`` you can specify the command line arguments for ``./configure`` by implementing ``configure_args``: .. code-block:: python - + class MyPkg(AutotoolsPackage): def configure_args(self): # FIXME: Add arguments other than --prefix @@ -96,7 +96,7 @@ From simplest to most complex, the following are the most common ways to customi Similarly for ``CMakePackage`` you can influence how ``cmake`` is invoked by implementing ``cmake_args``: .. code-block:: python - + class MyPkg(CMakePackage): def cmake_args(self): # FIXME: Add arguments other than @@ -113,7 +113,7 @@ From simplest to most complex, the following are the most common ways to customi You can set these variables by overriding the ``setup_build_environment`` method in your package class: .. code-block:: python - + def setup_build_environment(self, env): env.set("MY_ENV_VAR", "value") @@ -126,7 +126,7 @@ From simplest to most complex, the following are the most common ways to customi This is useful for installing additional files missed by the build system, or for running custom scripts. .. code-block:: python - + @run_after("install") def install_missing_files(self): install_tree("extra_files", self.prefix.bin) @@ -147,9 +147,9 @@ In any of the functions above, you can if self.spec.satisfies("+variant_name"): ... - + to check if a variant is enabled, or - + .. code-block:: python self.spec["dependency_name"].prefix @@ -363,7 +363,7 @@ This example adds a flag when the C compiler is from GCC version 8 or higher. The ``%c=gcc`` syntax technically means that ``gcc`` is the provider for the ``c`` language virtual. .. tip:: - + Historically, many packages have been written using ``^dep`` to refer to a dependency. Modern Spack packages should consider using ``%dep`` instead, which is more precise: it can only match direct dependencies, which are listed in the ``depends_on`` statements. @@ -456,20 +456,20 @@ We can get the provider's (e.g. OpenBLAS or Intel MKL) prefixes like this: f"--with-lapack={self.spec['lapack'].prefix}", ] -Many build systems struggle to locate the ``blas`` and ``lapack`` libraries during configure, either because they do not know the exact names of the libraries, or because the libraries are not in typical locations --- they may not even know whether blas and lapack are a single or separate libraries. +Many build systems struggle to locate the ``blas`` and ``lapack`` libraries during configure, either because they do not know the exact names of the libraries, or because the libraries are not in typical locations --- they may not even know whether ``blas`` and ``lapack`` are a single or separate libraries. In those cases, the build system could use some help, for which we give a few examples below: 1. Space separated list of full paths .. code-block:: python - + lapack_blas = spec["lapack"].libs + spec["blas"].libs args.append(f"--with-blas-lapack-lib={lapack_blas.joined()}") 2. Names of libraries and directories which contain them .. code-block:: python - + lapack_blas = spec["lapack"].libs + spec["blas"].libs args.extend( [ @@ -481,7 +481,7 @@ In those cases, the build system could use some help, for which we give a few ex 3. Search and link flags .. code-block:: python - + lapack_blas = spec["lapack"].libs + spec["blas"].libs args.append(f"-DMATH_LIBS={lapack_blas.ld_flags}") @@ -610,7 +610,7 @@ Not all dependencies set up such variables for dependent packages, in which case 1. Use the ``command`` attribute of the dependency. This is a good option, since it refers to an executable provided by a specific dependency. - + .. code-block:: python def install(self, spec: Spec, prefix: Prefix) -> None: @@ -619,7 +619,7 @@ Not all dependencies set up such variables for dependent packages, in which case 2. Use the ``which`` function (from the ``spack.package`` module). Do note that this function relies on the order of the ``PATH`` environment variable, which may be less reliable than the first option. - + .. code-block:: python def install(self, spec: Spec, prefix: Prefix) -> None: @@ -1151,7 +1151,7 @@ The ``compiler-wrapper`` package has several responsibilities: * It sets the ``CC``, ``CXX``, and ``FC`` environment variables in the :ref:`build environment `. These variables point to a wrapper executable in the ``compiler-wrapper``'s bin directory, which is a shell script that ultimately invokes the actual, underlying compiler executable. * It ensures that three kinds of compiler flags are passed to the compiler when it is invoked: - + 1. Flags requested by the user and package author (see :ref:`compiler flags `) 2. Flags needed to locate headers and libraries (during the build as well as at runtime) 3. Target specific flags, like ``-march=x86-64-v3``, translated from the spec's ``target=`` variant. @@ -1198,7 +1198,7 @@ Spack heavily makes use of `RPATHs `_ on Lin Executables are able to find their needed libraries *without* any of the infamous environment variables such as ``LD_LIBRARY_PATH`` on Linux or ``DYLD_LIBRARY_PATH`` on macOS. The :ref:`compiler wrapper ` is the main component that ensures that all binaries built by Spack have the correct RPATHs set. -As a package author, you rarely need to worry about RPATHs: the relevant compiler flags are automatically injected through the compiler wrappers, and the build system is blisfully unaware of them. +As a package author, you rarely need to worry about RPATHs: the relevant compiler flags are automatically injected through the compiler wrappers, and the build system is blissfully unaware of them. This works for most packages and build systems, with the notable exception of CMake, which has its own RPATH handling. CMake has its own RPATH handling, and distinguishes between build and install RPATHs. @@ -1260,7 +1260,7 @@ Loosely, there are three types of MPI builds: 3. CMake's ``FindMPI`` needs the compiler wrappers, but it uses them to extract ``-I`` / ``-L`` / ``-D`` arguments, then treats MPI like a regular library. Note that some CMake builds fall into case 2 because they either don't know about or don't like CMake's ``FindMPI`` support -- they just assume an MPI compiler. -Also, some autotools builds fall into case 3 (e.g., `here is an autotools version of CMake's FindMPI `_). +Also, some Autotools builds fall into case 3 (e.g., `here is an autotools version of CMake's FindMPI `_). Given all of this, we leave the use of the wrappers up to the packager. Spack will support all three ways of building MPI packages. @@ -1298,11 +1298,11 @@ So using the MPI wrappers should really be as simple as the code above. ``spec["mpi"]`` ^^^^^^^^^^^^^^^^^^^^^ -Ok, so how does all this work? +Okay, so how does all this work? If your package has a virtual dependency like ``mpi``, then referring to ``spec["mpi"]`` within ``install()`` will get you the concrete ``mpi`` implementation in your dependency DAG. That is a spec object just like the one passed to install, only the MPI implementations all set some additional properties on it to help you out. -E.g., in openmpi, you'll find this: +E.g., in ``openmpi``, you'll find this: .. literalinclude:: .spack/spack-packages/repos/spack_repo/builtin/packages/openmpi/package.py :pyobject: Openmpi.setup_dependent_package @@ -1318,13 +1318,13 @@ Wrapping wrappers Spack likes to use its own compiler wrappers to make it easy to add ``RPATHs`` to builds, and to try hard to ensure that your builds use the right dependencies. This doesn't play nicely by default with MPI, so we have to do a couple of tricks. -1. If we build MPI with Spack's wrappers, mpicc and friends will be installed with hard-coded paths to Spack's wrappers, and using them from outside of Spack will fail because they only work within Spack. - To fix this, we patch mpicc and friends to use the regular compilers. - Look at the filter_compilers method in mpich, openmpi, or mvapich2 for details. +1. If we build MPI with Spack's wrappers, ``mpicc`` and friends will be installed with hard-coded paths to Spack's wrappers, and using them from outside of Spack will fail because they only work within Spack. + To fix this, we patch ``mpicc`` and friends to use the regular compilers. + Look at the filter_compilers method in ``mpich``, ``openmpi``, or ``mvapich2`` for details. -2. We still want to use the Spack compiler wrappers when Spack is calling mpicc. - Luckily, wrappers in all mainstream MPI implementations provide environment variables that allow us to dynamically set the compiler to be used by mpicc, mpicxx, etc. - Spack's build environment sets ``MPICC``, ``MPICXX``, etc. for mpich derivatives and ``OMPI_CC``, ``OMPI_CXX``, etc. for OpenMPI. +2. We still want to use the Spack compiler wrappers when Spack is calling ``mpicc``. + Luckily, wrappers in all mainstream MPI implementations provide environment variables that allow us to dynamically set the compiler to be used by ``mpicc``, ``mpicxx``, etc. + Spack's build environment sets ``MPICC``, ``MPICXX``, etc. for MPICH derivatives and ``OMPI_CC``, ``OMPI_CXX``, etc. for OpenMPI. This makes the MPI compiler wrappers use the Spack compiler wrappers so that your dependencies still get proper RPATHs even if you use the MPI wrappers. MPI on Cray machines diff --git a/lib/spack/docs/packaging_guide_creation.rst b/lib/spack/docs/packaging_guide_creation.rst index c4669e89e11560..95f5af4fd21d83 100644 --- a/lib/spack/docs/packaging_guide_creation.rst +++ b/lib/spack/docs/packaging_guide_creation.rst @@ -1533,6 +1533,8 @@ In this case, examples of valid options are ``process_managers=auto``, ``process Both validator functions return a :py:class:`~spack.variant.DisjointSetsOfValues` object, which defines chaining methods to further customize the behavior of the variant. +.. _variant-conditional-values: + Conditional Possible Values ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1660,7 +1662,7 @@ Let's take a look at the ``libdwarf`` package to see how it's done: ^^^^^^^^^^^^^^^^ The highlighted ``depends_on("libelf")`` call tells Spack that it needs to build and install the ``libelf`` package before it builds ``libdwarf``. -This means that in your ``install()`` method, you are guaranteed that ``libelf`` has been built and installed successfully, so you can rely on it for your libdwarf build. +This means that in your ``install()`` method, you are guaranteed that ``libelf`` has been built and installed successfully, so you can rely on it for your ``libdwarf`` build. .. _dependency_specs: @@ -1701,7 +1703,7 @@ Spack allows you to specify this in the ``depends_on`` directive using version r depends_on("python@3.10:") -In this case, the package requires Python 3.10 or newer. +In this case, the package requires Python 3.10 or newer, as specified in the project's :file:`pyproject.toml`. Commonly, packages drop support for older versions of a dependency as they release new versions. In Spack you can conveniently add every backward compatibility rule as a separate line: @@ -1768,6 +1770,22 @@ For example, if you need Boost 1.59.0 or newer, but there are known issues with depends_on("boost@1.59.0:1.63,1.65.1,1.67.0:") +or, if those particular versions are excluded due to bugs rather than removed and reintroduced features: + +.. code-block:: python + + depends_on("boost@1.59.0:") + conflicts("^boost@1.64.0,1.65.0,1.66.0") + +Always specify version ranges with an open-world assumption: + +- all "ground truths" about exclusions and inclusions (e.g., versions with features added or removed) must satisfy the range, and +- no potential but unknown versions are excluded from the range. + +This practice avoids overconstraining version ranges, which can lead to concretization errors, and ensures that every version in a package is *meaningful* and not just *incidental* (i.e., based on the version you happened to test). +In the above example, the project has presumably documented (with pyproject.toml, CMakeLists.txt, or release notes) that ``@:1.58`` are incompatible, and it is known from testing that ``@1.67`` is compatible. +It is *not* known whether future versions ``@1.68:`` are incompatible, so they must be included by the range. +If and when future versions are known incompatible, the version range should be constrained with an upper bound. .. _dependency-types: @@ -2294,7 +2312,7 @@ Only needed for patches fetched from URLs. If supplied, this is a spec that tells Spack when to apply the patch. If the installed package spec matches this spec, the patch will be applied. -In our example above, the patch is applied when mvapich is at version ``1.9`` or higher. +In our example above, the patch is applied when ``mvapich`` is at version ``1.9`` or higher. ``level`` """"""""" @@ -2325,7 +2343,7 @@ Lines 1-2 show paths with synthetic ``a/`` and ``b/`` prefixes. These are placeholders for the two ``mvapich2`` source directories that ``diff`` compared when it created the patch file. This is git's default behavior when creating patch files, but other programs may behave differently. -``-p1`` strips off the first level of the prefix in both paths, allowing the patch to be applied from the root of an expanded mvapich2 archive. +``-p1`` strips off the first level of the prefix in both paths, allowing the patch to be applied from the root of an expanded ``mvapich2`` archive. If you set level to ``2``, it would strip off ``src``, and so on. It's generally easier to just structure your patch file so that it applies cleanly with ``-p1``, but if you're using a patch you didn't create yourself, ``level`` can be handy. @@ -2482,7 +2500,7 @@ This ensures that Python in a view can always locate its Python packages, even w A package can only extend one other package at a time. To support packages that may extend one of a list of other packages, Spack supports multiple ``extends`` directives as long as at most one of them is selected as a dependency during concretization. -For example, a lua package could extend either lua or luajit, but not both: +For example, a lua package could extend either ``lua`` or ``lua-luajit``, but not both: .. code-block:: python @@ -2493,7 +2511,7 @@ For example, a lua package could extend either lua or luajit, but not both: extends("lua-luajit", when="~use_lua") ... -Now, a user can install, and activate, the ``lua-lpeg`` package for either lua or luajit. +Now, a user can install, and activate, the ``lua-lpeg`` package for either lua or ``lua-luajit``. Adding additional constraints ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/docs/packaging_guide_testing.rst b/lib/spack/docs/packaging_guide_testing.rst index 0d8f6516da2c33..a4bc2a8b60b65f 100644 --- a/lib/spack/docs/packaging_guide_testing.rst +++ b/lib/spack/docs/packaging_guide_testing.rst @@ -49,7 +49,7 @@ Success is assumed if anything (e.g., a file or directory) is written after ``in Otherwise, the build is assumed to have failed. However, the presence of install prefix contents is not a sufficient indicator of success so Spack supports the addition of tests that can be performed during `spack install` processing. -Consider a simple autotools build using the following commands: +Consider a simple Autotools build using the following commands: .. code-block:: console diff --git a/lib/spack/docs/pipelines.rst b/lib/spack/docs/pipelines.rst index befee35c0f4f28..91ee717926010a 100644 --- a/lib/spack/docs/pipelines.rst +++ b/lib/spack/docs/pipelines.rst @@ -87,7 +87,7 @@ Here's the ``.gitlab-ci.yml`` file from that example that builds and runs the pi job: generate-pipeline -The key thing to note above is that there are two jobs: The first job to run, ``generate-pipeline``, runs the ``spack ci generate`` command to generate a dynamic child pipeline and write it to a yaml file, which is then picked up by the second job, ``build-jobs``, and used to trigger the downstream pipeline. +The key thing to note above is that there are two jobs: The first job to run, ``generate-pipeline``, runs the ``spack ci generate`` command to generate a dynamic child pipeline and write it to a YAML file, which is then picked up by the second job, ``build-jobs``, and used to trigger the downstream pipeline. And here's the Spack environment built by the pipeline represented as a ``spack.yaml`` file: @@ -161,7 +161,7 @@ This file, ``mirrors.yaml`` looks like this: Note the name of the mirror is ``buildcache-destination``, which is required as of Spack 0.23 (see below for more information). -The mirror url simply points to the container registry associated with the project, while ``id_variable`` and ``secret_variable`` refer to environment variables containing the access credentials for the mirror. +The mirror URL simply points to the container registry associated with the project, while ``id_variable`` and ``secret_variable`` refer to environment variables containing the access credentials for the mirror. When Spack builds packages for this example project, they will be pushed to the project container registry, where they will be available for subsequent jobs to install as dependencies or for other pipelines to use to build runnable container images. @@ -184,9 +184,8 @@ Super-command for functionality related to generating pipelines and executing pi ^^^^^^^^^^^^^^^^^^^^^ Throughout this documentation, references to the "mirror" mean the target mirror which is checked for the presence of up-to-date specs, and where any scheduled jobs should push built binary packages. -In the past, this defaulted to the mirror at index 0 in the mirror configs, and could be overridden using the ``--buildcache-destination`` argument. -Starting with Spack 0.23, ``spack ci generate`` will require you to identify this mirror by the name "buildcache-destination". -While you can configure any number of mirrors as sources for your pipelines, you will need to identify the destination mirror by name. +When running ``spack ci generate`` it is required to configure a mirror named ``buildcache-destination`` to be used as the target mirror. +It is permitted to configure any number of other mirrors as sources for your pipelines, but only the ``buildcache-destination`` mirror will be used as the destination mirror. Concretizes the specs in the active environment, stages them (as described in :ref:`staging_algorithm`), and writes the resulting ``.gitlab-ci.yml`` to disk. During concretization of the environment, ``spack ci generate`` also writes a ``spack.lock`` file which is then provided to generated child jobs and made available in all generated job artifacts to aid in reproducing failed builds in a local environment. @@ -197,9 +196,9 @@ In the :ref:`functional_example` section, we only mentioned one path in the ``ar Using ``--prune-dag`` or ``--no-prune-dag`` configures whether or not jobs are generated for specs that are already up to date on the mirror. If enabling DAG pruning using ``--prune-dag``, more information may be required in your ``spack.yaml`` file, see the :ref:`noop_jobs` section below regarding ``noop-job``. -The optional ``--check-index-only`` argument can be used to speed up pipeline generation by telling Spack to consider only remote buildcache indices when checking the remote mirror to determine if each spec in the DAG is up to date or not. +The optional ``--check-index-only`` argument can be used to speed up pipeline generation by telling Spack to consider only remote build cache indices when checking the remote mirror to determine if each spec in the DAG is up to date or not. The default behavior is for Spack to fetch the index and check it, but if the spec is not found in the index, it also performs a direct check for the spec on the mirror. -If the remote buildcache index is out of date, which can easily happen if it is not updated frequently, this behavior ensures that Spack has a way to know for certain about the status of any concrete spec on the remote mirror, but can slow down pipeline generation significantly. +If the remote build cache index is out of date, which can easily happen if it is not updated frequently, this behavior ensures that Spack has a way to know for certain about the status of any concrete spec on the remote mirror, but can slow down pipeline generation significantly. The optional ``--output-file`` argument should be an absolute path (including file name) to the generated pipeline, and if not given, the default is ``./.gitlab-ci.yml``. @@ -264,7 +263,7 @@ You can find them under Spack's `stacks `_ for the ci section of the Spack environment file, to see precisely what syntax is allowed there. +Take a look at the `schema `_ for the ``ci`` section of the Spack environment file, to see precisely what syntax is allowed there. .. _reserved_tags: @@ -702,6 +701,6 @@ Only needed to report build groups to CDash. ^^^^^^^^^^^^^^^^^^^^^ Optional. -Only needed if you want ``spack ci rebuild`` to trust the key you store in this variable, in which case, it will subsequently be used to sign and verify binary packages (when installing or creating buildcaches). -You could also have already trusted a key Spack knows about, or if no key is present anywhere, Spack will install specs using ``--no-check-signature`` and create buildcaches using ``-u`` (for unsigned binaries). +Only needed if you want ``spack ci rebuild`` to trust the key you store in this variable, in which case, it will subsequently be used to sign and verify binary packages (when installing or creating build caches). +You could also have already trusted a key Spack knows about, or if no key is present anywhere, Spack will install specs using ``--no-check-signature`` and create build caches using ``-u`` (for unsigned binaries). diff --git a/lib/spack/docs/requirements.txt b/lib/spack/docs/requirements.txt index 4422cce37178ee..1c70fbee0def04 100644 --- a/lib/spack/docs/requirements.txt +++ b/lib/spack/docs/requirements.txt @@ -1,9 +1,10 @@ sphinx==9.1.0 -sphinxcontrib-programoutput==0.18 -sphinxcontrib-svg2pdfconverter==2.0.0 +sphinxcontrib-programoutput==0.19 +sphinxcontrib-svg2pdfconverter==2.1.0 sphinx-copybutton==0.5.2 sphinx-last-updated-by-git==0.3.8 sphinx-sitemap==2.9.0 furo==2025.12.19 docutils==0.22.4 -pygments==2.19.2 +pygments==2.20.0 +pytest==9.0.3 diff --git a/lib/spack/docs/signing.rst b/lib/spack/docs/signing.rst index de34abeae57cb6..eb11dfbba4f1ed 100644 --- a/lib/spack/docs/signing.rst +++ b/lib/spack/docs/signing.rst @@ -217,7 +217,7 @@ Procedurally the ``spack-intermediate-ci-signing-key`` secret is used in the fol 1. A ``large-arm-prot`` or ``large-x86-prot`` protected runner picks up a job tagged ``protected`` from a protected GitLab branch. (See :ref:`protected_runners`). -2. Based on its configuration, the runner creates a job Pod in the pipeline namespace and mounts the spack-intermediate-ci-signing-key Kubernetes secret into the build container +2. Based on its configuration, the runner creates a job Pod in the pipeline namespace and mounts the ``spack-intermediate-ci-signing-key`` Kubernetes secret into the build container 3. The Intermediate CI Key, affiliated institutions' public key and the Reputational Public Key are imported into a keyring by the ``spack gpg ...`` sub-command. This is initiated by the job's build script which is created by the generate job at the beginning of the pipeline. 4. Assuming the package has dependencies those spec manifests are verified using the keyring. @@ -269,7 +269,7 @@ Protected Runners and Reserved Tags Spack has a large number of Gitlab Runners operating in its build farm. These include runners deployed in the AWS Kubernetes cluster as well as runners deployed at affiliated institutions. -The majority of runners are shared runners that operate across projects in gitlab.spack.io. +The majority of runners are shared runners that operate across projects in `gitlab.spack.io `_. These runners pick up jobs primarily from the spack/spack project and execute them in PR pipelines. A small number of runners operating on AWS and at affiliated institutions are registered as specific *protected* runners on the spack/spack project. diff --git a/lib/spack/docs/spec_syntax.rst b/lib/spack/docs/spec_syntax.rst index 7cdd9af21b068f..27396b9c78f657 100644 --- a/lib/spack/docs/spec_syntax.rst +++ b/lib/spack/docs/spec_syntax.rst @@ -18,7 +18,7 @@ Spack uses specs to: 1. Refer to a particular build configuration of a package, or 2. Express requirements, or preferences, on packages via configuration files, or -3. Query installed packages, or buildcaches +3. Query installed packages, or build caches Specs are more than a package name and a version; you can use them to specify the compiler, compiler version, architecture, compile options, and dependency options for a build. In this section, we'll go over the full syntax of specs. @@ -644,7 +644,7 @@ To work around this without quoting, you can avoid whitespace between the packag mpileaks ~debug # shell may expand this to `mpileaks /home/debug` mpileaks~debug # use this instead - + Alternatively, you can use a hyphen ``-`` character to disable a variant, but be aware that this *requires* a space between the package name and the variant: .. code-block:: spec diff --git a/lib/spack/docs/toolchains_yaml.rst b/lib/spack/docs/toolchains_yaml.rst index 4d322cf69082bc..e0b6a934cef5b6 100644 --- a/lib/spack/docs/toolchains_yaml.rst +++ b/lib/spack/docs/toolchains_yaml.rst @@ -45,7 +45,7 @@ The spec ``cflags=-O3`` is *always* applied, because there is no ``when`` clause The toolchain can be referenced using .. code-block:: spec - + $ spack install my-package %llvm_gfortran Toolchains are useful for three reasons: diff --git a/lib/spack/docs/windows.rst b/lib/spack/docs/windows.rst index 178cafcd4cefab..843d705b1686a0 100644 --- a/lib/spack/docs/windows.rst +++ b/lib/spack/docs/windows.rst @@ -106,7 +106,7 @@ In order to install Spack with Windows support, run the following one-liner in a Step 3: Run and configure Spack ------------------------------- -On Windows, Spack supports both primary native shells, Powershell and the traditional command prompt. +On Windows, Spack supports both primary native shells, PowerShell and the traditional command prompt. To use Spack, pick your favorite shell, and run ``bin\spack_cmd.bat`` or ``share/spack/setup-env.ps1`` (you may need to Run as Administrator) from the top-level Spack directory. This will provide a Spack-enabled shell. If you receive a warning message that Python is not in your ``PATH`` (which may happen if you installed Python from the website and not the Windows Store), add the location of the Python executable to your ``PATH`` now. diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 0c25558b512a8b..ae772db4099df4 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import functools import os import re from typing import Optional @@ -18,7 +19,7 @@ #: version is incremented when the package API is extended in a backwards-compatible way. The major #: version is incremented upon breaking changes. This version is changed independently from the #: Spack version. -package_api_version = (2, 4) +package_api_version = (2, 5) #: The minimum Package API version that this version of Spack is compatible with. This should #: always be a tuple of the form ``(major, 0)``, since compatibility with vX.Y implies @@ -37,6 +38,7 @@ def __try_int(v): spack_version_info = tuple([__try_int(v) for v in __version__.split(".")]) +@functools.lru_cache(maxsize=None) def get_spack_commit() -> Optional[str]: """Get the Spack git commit sha. diff --git a/lib/spack/spack/audit.py b/lib/spack/spack/audit.py index 0efad6291d8653..21919b94daf97d 100644 --- a/lib/spack/spack/audit.py +++ b/lib/spack/spack/audit.py @@ -34,6 +34,7 @@ def _search_duplicate_compilers(error_cls): the decorator object, that will forward the keyword arguments passed as input. """ + import ast import collections import collections.abc @@ -51,6 +52,7 @@ def _search_duplicate_compilers(error_cls): import spack.builder import spack.config +import spack.enums import spack.fetch_strategy import spack.llnl.util.lang import spack.patch @@ -385,15 +387,6 @@ def _make_config_error(config_data, summary, error_cls): kwargs=("pkgs",), ) - -package_deprecated_attributes = AuditClass( - group="packages", - tag="PKG-DEPRECATED-ATTRIBUTES", - description="Sanity checks to preclude use of deprecated package attributes", - kwargs=("pkgs",), -) - - package_properties = AuditClass( group="packages", tag="PKG-PROPERTIES", @@ -434,6 +427,22 @@ def _check_build_test_callbacks(pkgs, error_cls): return errors +@package_directives +def _directives_can_be_evaluated(pkgs, error_cls): + """Ensure that all directives in a package can be evaluated.""" + errors = [] + for pkg_name in pkgs: + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) + for attr in pkg_cls._dict_to_directives: + try: + getattr(pkg_cls, attr) + except Exception as e: + error_msg = f"Package '{pkg_name}' has invalid directive '{attr}'" + details = [str(e)] + errors.append(error_cls(error_msg, details)) + return errors + + @package_directives def _check_patch_urls(pkgs, error_cls): """Ensure that patches fetched from GitHub and GitLab have stable sha256 @@ -526,46 +535,6 @@ def _search_for_reserved_attributes_names_in_packages(pkgs, error_cls): return errors -@package_deprecated_attributes -def _search_for_deprecated_package_methods(pkgs, error_cls): - """Ensure the package doesn't define or use deprecated methods""" - DEPRECATED_METHOD = (("test", "a name starting with 'test_'"),) - DEPRECATED_USE = ( - ("self.cache_extra_test_sources(", "cache_extra_test_sources(self, ..)"), - ("self.install_test_root(", "install_test_root(self, ..)"), - ("self.run_test(", "test_part(self, ..)"), - ) - errors = [] - for pkg_name in pkgs: - pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) - methods = inspect.getmembers(pkg_cls, predicate=lambda x: inspect.isfunction(x)) - method_errors = collections.defaultdict(list) - for name, function in methods: - for deprecated_name, alternate in DEPRECATED_METHOD: - if name == deprecated_name: - msg = f"Rename '{deprecated_name}' method to {alternate} instead." - method_errors[name].append(msg) - - source = inspect.getsource(function) - for deprecated_name, alternate in DEPRECATED_USE: - if deprecated_name in source: - msg = f"Change '{deprecated_name}' to '{alternate}' in '{name}' method." - method_errors[name].append(msg) - - num_methods = len(method_errors) - if num_methods > 0: - methods = plural(num_methods, "method", show_n=False) - error_msg = ( - f"Package '{pkg_name}' implements or uses unsupported deprecated {methods}." - ) - instr = [f"Make changes to '{pkg_cls.__module__}':"] - for name in sorted(method_errors): - instr.extend([f" {msg}" for msg in method_errors[name]]) - errors.append(error_cls(error_msg, instr)) - - return errors - - @package_properties def _ensure_all_package_names_are_lowercase(pkgs, error_cls): """Ensure package names are lowercase and consistent""" @@ -921,75 +890,104 @@ def _linting_package_file(pkgs, error_cls): # Does the homepage have http, and if so, does https work? if homepage.startswith("http://"): try: - response = urlopen(f"https://{homepage[7:]}") + with urlopen(f"https://{homepage[7:]}") as response: + if response.getcode() == 200: + msg = 'Package "{0}" uses http but has a valid https endpoint.' + errors.append(msg.format(pkg_cls.name)) except Exception as e: msg = 'Error with attempting https for "{0}": ' errors.append(error_cls(msg.format(pkg_cls.name), [str(e)])) continue - if response.getcode() == 200: - msg = 'Package "{0}" uses http but has a valid https endpoint.' - errors.append(msg.format(pkg_cls.name)) - return spack.llnl.util.lang.dedupe(errors) @package_directives -def _unknown_variants_in_directives(pkgs, error_cls): - """Report unknown or wrong variants in directives for this package""" +def _variant_issues_in_directives(pkgs, error_cls): + """Report unknown, wrong, or propagating variants in directives for this package""" errors = [] for pkg_name in pkgs: pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) + filename = spack.repo.PATH.filename_for_package_name(pkg_name) - # Check "conflicts" directive + # Check the "conflicts" directive for trigger, conflicts in pkg_cls.conflicts.items(): + errors.extend( + _issues_in_directive_constraint( + pkg_cls, + spack.spec.Spec(trigger), + directive="conflicts", + error_cls=error_cls, + filename=filename, + requestor=pkg_name, + ) + ) for conflict, _ in conflicts: - vrn = spack.spec.Spec(conflict) - try: - vrn.constrain(trigger) - except Exception: - # If one of the conflict/trigger includes a platform and the other - # includes an os or target, the constraint will fail if the current - # platform is not the plataform in the conflict/trigger. Audit the - # conflict and trigger separately in that case. - # When os and target constraints can be created independently of - # the platform, TODO change this back to add an error. - errors.extend( - _analyze_variants_in_directive( - pkg_cls, - spack.spec.Spec(trigger), - directive="conflicts", - error_cls=error_cls, - ) - ) errors.extend( - _analyze_variants_in_directive( - pkg_cls, vrn, directive="conflicts", error_cls=error_cls + _issues_in_directive_constraint( + pkg_cls, + spack.spec.Spec(conflict), + directive="conflicts", + error_cls=error_cls, + filename=filename, + requestor=pkg_name, ) ) # Check "depends_on" directive - for trigger in pkg_cls.dependencies: + for trigger, deps_by_name in pkg_cls.dependencies.items(): vrn = spack.spec.Spec(trigger) errors.extend( - _analyze_variants_in_directive( - pkg_cls, vrn, directive="depends_on", error_cls=error_cls + _issues_in_directive_constraint( + pkg_cls, + vrn, + directive="depends_on", + error_cls=error_cls, + filename=filename, + requestor=pkg_name, ) ) + for dep_name, dep in deps_by_name.items(): + if spack.repo.PATH.is_virtual(dep_name): + continue + try: + dep_pkg_cls = spack.repo.PATH.get_pkg_class(dep_name) + except spack.repo.UnknownPackageError: + continue + errors.extend( + _issues_in_directive_constraint( + dep_pkg_cls, + dep.spec, + directive="depends_on", + error_cls=error_cls, + filename=filename, + requestor=pkg_name, + ) + ) # Check "provides" directive for when_spec in pkg_cls.provided: errors.extend( - _analyze_variants_in_directive( - pkg_cls, when_spec, directive="provides", error_cls=error_cls + _issues_in_directive_constraint( + pkg_cls, + when_spec, + directive="provides", + error_cls=error_cls, + filename=filename, + requestor=pkg_name, ) ) # Check "resource" directive for vrn in pkg_cls.resources: errors.extend( - _analyze_variants_in_directive( - pkg_cls, vrn, directive="resource", error_cls=error_cls + _issues_in_directive_constraint( + pkg_cls, + vrn, + directive="resource", + error_cls=error_cls, + filename=filename, + requestor=pkg_name, ) ) @@ -1190,18 +1188,66 @@ def _version_constraints_are_satisfiable_by_some_version_in_repo(pkgs, error_cls return errors -def _analyze_variants_in_directive(pkg, constraint, directive, error_cls): +@package_directives +def _ensure_maintainers_are_not_placeholders(pkgs, error_cls): + """Ensure placeholder maintainers are not defined in the package.""" errors = [] - variant_names = pkg.variant_names() - summary = f"{pkg.name}: wrong variant in '{directive}' directive" - filename = spack.repo.PATH.filename_for_package_name(pkg.name) + placeholder_maintainers = ("github_user1", "github_user2") + for pkg_name in pkgs: + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) + found_placeholders = set(pkg_cls.maintainers).intersection(placeholder_maintainers) + + if found_placeholders: + summary = f"Package '{pkg_name}' has placeholder maintainer(s)" + details = [f"Remove placeholder maintainer(s): {found_placeholders}"] + errors.append(error_cls(summary, details)) + return errors + +def _issues_in_directive_constraint(pkg, constraint, *, directive, error_cls, filename, requestor): + errors = [] + errors.extend( + _analyze_variants_in_directive( + pkg, + constraint, + directive=directive, + error_cls=error_cls, + filename=filename, + requestor=requestor, + ) + ) + errors.extend( + _analize_propagated_deps_in_directive( + pkg, + constraint, + directive=directive, + error_cls=error_cls, + filename=filename, + requestor=requestor, + ) + ) + return errors + + +def _analyze_variants_in_directive(pkg, constraint, *, directive, error_cls, filename, requestor): + errors = [] + variant_names = pkg.variant_names() + summary = f"{requestor}: wrong variant in '{directive}' directive" for name, v in constraint.variants.items(): + if name == "commit": + # Automatic variant + continue + if name not in variant_names: msg = f"variant {name} does not exist in {pkg.name}" errors.append(error_cls(summary=summary, details=[msg, f"in {filename}"])) continue + if v.propagate: + propagation_summary = f"{requestor}: propagating variant in '{directive}' directive" + msg = f"using {constraint} in a directive, which propagates the '{name}' variant" + errors.append(error_cls(summary=propagation_summary, details=[msg, f"in {filename}"])) + try: spack.variant.prevalidate_variant_value(pkg, v, constraint, strict=True) except ( @@ -1215,6 +1261,18 @@ def _analyze_variants_in_directive(pkg, constraint, directive, error_cls): return errors +def _analize_propagated_deps_in_directive( + pkg, constraint, *, directive, error_cls, filename, requestor +): + errors = [] + summary = f"{requestor}: dependency propagation ('%%') in '{directive}' directive" + for edge in constraint.traverse_edges(): + if edge.propagation != spack.enums.PropagationPolicy.NONE: + msg = f"'{edge.spec}' contains a propagated dependency" + errors.append(error_cls(summary=summary, details=[msg, f"in {filename}"])) + return errors + + @package_directives def _named_specs_in_when_arguments(pkgs, error_cls): """Reports named specs in the 'when=' attribute of a directive. diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index e04be69ba49d31..d42f6129a1130a 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -26,7 +26,21 @@ import warnings from collections import defaultdict from contextlib import closing -from typing import IO, Callable, Dict, Iterable, List, Mapping, Optional, Set, Tuple, Union +from typing import ( + IO, + Any, + Callable, + Dict, + Iterable, + List, + Mapping, + NamedTuple, + Optional, + Set, + Tuple, + Union, + cast, +) import spack.caches import spack.config @@ -97,6 +111,7 @@ get_url_buildcache_class, get_valid_spec_file, ) +from .vendor.typing_extensions import TypedDict class BuildCacheDatabase(spack.database.Database): @@ -144,9 +159,28 @@ def __init__(self, errors): super().__init__(self.message) -class BinaryCacheIndex: +class _MirrorIndexResult(NamedTuple): + succeeded: bool + regenerate: bool + had_cache_entry: bool + error: Optional[Exception] + no_index: bool = False + + +class _LastFetch(NamedTuple): + time: float + succeeded: bool + + +class _LocalIndexCache(TypedDict, total=False): + index_hash: str + index_path: str + etag: str + + +class BinaryIndexCache: """ - The BinaryCacheIndex tracks what specs are available on (usually remote) + The BinaryIndexCache tracks what specs are available on (usually remote) binary caches. This index is "best effort", in the sense that whenever we don't find @@ -170,7 +204,7 @@ def __init__(self, cache_root: Optional[str] = None): self._index_file_cache_initialized = False # stores a map of mirror URL and version layout to index hash and cache key (index path) - self._local_index_cache: dict[str, dict] = {} + self._local_index_cache: dict[str, _LocalIndexCache] = {} # hashes of remote indices already ingested into the concrete spec # cache (_mirrors_for_spec) @@ -178,23 +212,21 @@ def __init__(self, cache_root: Optional[str] = None): # mapping from mirror urls to the time.time() of the last index fetch and a bool indicating # whether the fetch succeeded or not. - self._last_fetch_times: Dict[MirrorMetadata, Tuple[float, bool]] = {} + self._last_fetch_times: Dict[MirrorMetadata, _LastFetch] = {} #: Dictionary mapping DAG hashes of specs to Spec objects self._known_specs: Dict[str, spack.spec.Spec] = {} #: Dictionary mapping DAG hashes of specs to a list of mirrors where they can be found self._mirrors_for_spec: Dict[str, Set[MirrorMetadata]] = defaultdict(set) + #: URLs of binary mirrors that had no buildcache index during the last update() + self.mirrors_without_index: Set[str] = set() def _init_local_index_cache(self): if not self._index_file_cache_initialized: cache_key = self._index_contents_key - self._index_file_cache.init_entry(cache_key) - - cache_path = self._index_file_cache.cache_path(cache_key) - self._local_index_cache = {} - if os.path.isfile(cache_path): - with self._index_file_cache.read_transaction(cache_key) as cache_file: + with self._index_file_cache.read_transaction(cache_key) as cache_file: + if cache_file is not None: self._local_index_cache = json.load(cache_file) self._index_file_cache_initialized = True @@ -231,23 +263,22 @@ def _associate_built_specs_with_mirror(self, cache_key, mirror_metadata: MirrorM with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir: db = BuildCacheDatabase(tmpdir) - try: - self._index_file_cache.init_entry(cache_key) - cache_path = self._index_file_cache.cache_path(cache_key) - with self._index_file_cache.read_transaction(cache_key): - db._read_from_file(pathlib.Path(cache_path)) - except spack.database.InvalidDatabaseVersionError as e: - tty.warn( - "you need a newer Spack version to read the buildcache index for the " - f"following v{mirror_metadata.version} mirror: '{mirror_metadata.url}'. " - f"{e.database_version_message}" - ) - return + with self._index_file_cache.read_transaction(cache_key) as f: + if f is not None: + try: + db._read_from_stream(f) + except spack.database.InvalidDatabaseVersionError as e: + tty.warn( + "you need a newer Spack version to read the buildcache index for the " + f"following v{mirror_metadata.version} mirror: " + f"'{mirror_metadata.url}'. {e.database_version_message}" + ) + return spec_list = [ s for s in db.query_local(installed=InstallRecordStatus.ANY) - # todo, make it easer to get install records associated with specs + # todo, make it easier to get install records associated with specs if s.external or db._data[s.dag_hash()].in_buildcache ] @@ -307,150 +338,118 @@ def update(self, with_cooldown: bool = False) -> None: from each configured mirror and stored locally (both in memory and on disk under ``_index_cache_root``).""" self._init_local_index_cache() - configured_mirrors = [ - MirrorMetadata(m.fetch_url, layout_version, m.fetch_view) + self.mirrors_without_index = set() + + supported_mirror_versions = { + (m.fetch_url, m.fetch_view): m.supported_layout_versions for m in spack.mirrors.mirror.MirrorCollection(binary=True).values() - for layout_version in m.supported_layout_versions - ] - items_to_remove = [] - spec_cache_clear_needed = False - spec_cache_regenerate_needed = not self._mirrors_for_spec - - # First compare the mirror urls currently present in the cache to the - # configured mirrors. If we have a cached index for a mirror which is - # no longer configured, we should remove it from the cache. For any - # cached indices corresponding to currently configured mirrors, we need - # to check if the cache is still good, or needs to be updated. - # Finally, if there are configured mirrors for which we don't have a - # cache entry, we need to fetch and cache the indices from those - # mirrors. - - # If, during this process, we find that any mirrors for which we - # already have entries have either been removed, or their index - # hash has changed, then our concrete spec cache (_mirrors_for_spec) - # likely has entries that need to be removed, so we will clear it - # and regenerate that data structure. - - # If, during this process, we find that there are new mirrors for - # which do not yet have an entry in our index cache, then we simply - # need to regenerate the concrete spec cache, but do not need to - # clear it first. - - # Otherwise the concrete spec cache should not need to be updated at - # all. - - fetch_errors: List[Exception] = [] - all_methods_failed = True - ttl = spack.config.get("config:binary_index_ttl", 600) - now = time.time() + } - for local_index_cache_key in self._local_index_cache: - urlAndVersion = MirrorMetadata.from_string(local_index_cache_key) - cached_mirror_url = urlAndVersion.url - cache_entry = self._local_index_cache[local_index_cache_key] - cached_index_path = cache_entry["index_path"] - if urlAndVersion in configured_mirrors: - # Only do a fetch if the last fetch was longer than TTL ago - if ( - with_cooldown - and ttl > 0 - and cached_mirror_url in self._last_fetch_times - and now - self._last_fetch_times[urlAndVersion][0] < ttl - ): - # We're in the cooldown period, don't try to fetch again - # If the fetch succeeded last time, consider this update a success, otherwise - # re-report the error here - if self._last_fetch_times[urlAndVersion][1]: - all_methods_failed = False - else: - # May need to fetch the index and update the local caches - needs_regen = False - try: - needs_regen = self._fetch_and_cache_index( - urlAndVersion, cache_entry=cache_entry - ) - self._last_fetch_times[urlAndVersion] = (now, True) - all_methods_failed = False - except FetchIndexError as e: - fetch_errors.append(e) - self._last_fetch_times[urlAndVersion] = (now, False) - except BuildcacheIndexNotExists: # as e: - # DH* JCSDA fork only - silence warnings about non-existent - # index for mirrors, because this warning is issued for all - # mirrors (source, bootstrap, ...) - # fetch_errors.append(e) - # *DH - self._last_fetch_times[urlAndVersion] = (now, False) - # Binary caches are not required to have an index, don't raise - # if it doesn't exist. - all_methods_failed = False - - # The need to regenerate implies a need to clear as well. - spec_cache_clear_needed |= needs_regen - spec_cache_regenerate_needed |= needs_regen - else: - # No longer have this mirror, cached index should be removed - items_to_remove.append( - { - "url": local_index_cache_key, - "cache_key": os.path.join(self._index_cache_root, cached_index_path), - } + # If we have a cached index for a mirror which is no longer configured, remove it + clear_cache, regenerate_cache = self._remove_stale_cache_entries(supported_mirror_versions) + + # Fetch or update the other indexes + errors, all_failed = [], True + for (url, view), versions in supported_mirror_versions.items(): + result = self._fetch_mirror_index(url, view, versions=versions, cooldown=with_cooldown) + if result.error: + errors.append(result.error) + + if result.succeeded: + all_failed = False + + if result.no_index: + self.mirrors_without_index.add(url) + + regenerate_cache |= result.regenerate + clear_cache |= result.regenerate and result.had_cache_entry + + self._write_local_index_cache() + + if supported_mirror_versions and all_failed: + raise FetchCacheError(errors) + + if errors: + warnings.warn( + "The following issues were ignored while updating the indices of binary caches:\n" + + str(FetchCacheError(errors)) + ) + + if regenerate_cache: + self.regenerate_spec_cache(clear_existing=clear_cache) + + def _fetch_mirror_index( + self, url: str, view: Optional[str], *, versions: List[int], cooldown: bool + ) -> _MirrorIndexResult: + """Fetches the index of a mirror, using a highest-version first approach, and returning + after the first success. + """ + now = time.time() + ttl = spack.config.CONFIG.get_config("config").get("binary_index_ttl", 600) + for version in versions: + meta = MirrorMetadata(url, version, view) + cache_entry = self._local_index_cache.get(str(meta)) + + if cache_entry is not None and ( + # Cache entry in cooldown + cooldown + and ttl > 0 + and meta in self._last_fetch_times + and now - self._last_fetch_times[meta].time < ttl + ): + return _MirrorIndexResult( + succeeded=self._last_fetch_times[meta].succeeded, + regenerate=False, + had_cache_entry=True, + error=None, ) - if urlAndVersion in self._last_fetch_times: - del self._last_fetch_times[urlAndVersion] - spec_cache_clear_needed = True - spec_cache_regenerate_needed = True - - # Clean up items to be removed, identified above - for item in items_to_remove: - url = item["url"] - cache_key = item["cache_key"] - self._index_file_cache.remove(cache_key) - del self._local_index_cache[url] - - # Iterate the configured mirrors now. Any mirror urls we do not - # already have in our cache must be fetched, stored, and represented - # locally. - for urlAndVersion in configured_mirrors: - if str(urlAndVersion) in self._local_index_cache: - continue - # Need to fetch the index and update the local caches - needs_regen = False try: - needs_regen = self._fetch_and_cache_index(urlAndVersion) - self._last_fetch_times[urlAndVersion] = (now, True) - all_methods_failed = False + regenerate = self._fetch_and_cache_index(meta, cache_entry=cache_entry or {}) + self._last_fetch_times[meta] = _LastFetch(time=now, succeeded=True) + return _MirrorIndexResult( + succeeded=True, + regenerate=regenerate, + had_cache_entry=cache_entry is not None, + error=None, + ) except FetchIndexError as e: - fetch_errors.append(e) - self._last_fetch_times[urlAndVersion] = (now, False) - except BuildcacheIndexNotExists: # as e: - # DH* JCSDA fork only - silence warnings about non-existent - # index for mirrors, because this warning is issued for all - # mirrors (source, bootstrap, ...) - # fetch_errors.append(e) - # *DH - self._last_fetch_times[urlAndVersion] = (now, False) - # Binary caches are not required to have an index, don't raise - # if it doesn't exist. - all_methods_failed = False - - # Generally speaking, a new mirror wouldn't imply the need to - # clear the spec cache, so leave it as is. - if needs_regen: - spec_cache_regenerate_needed = True + self._last_fetch_times[meta] = _LastFetch(time=now, succeeded=False) + return _MirrorIndexResult( + succeeded=False, + regenerate=False, + had_cache_entry=cache_entry is not None, + error=e, + ) + except BuildcacheIndexNotExists: + # Try next lower layout version + self._last_fetch_times[meta] = _LastFetch(time=now, succeeded=False) + continue - self._write_local_index_cache() + # All versions reported no index found. Record it for concretization callers to warn. + return _MirrorIndexResult( + succeeded=True, regenerate=False, had_cache_entry=False, error=None, no_index=True + ) - if configured_mirrors and all_methods_failed: - raise FetchCacheError(fetch_errors) - if fetch_errors: - tty.warn( - "The following issues were ignored while updating the indices of binary caches", - FetchCacheError(fetch_errors), - ) - if spec_cache_regenerate_needed: - self.regenerate_spec_cache(clear_existing=spec_cache_clear_needed) + def _remove_stale_cache_entries( + self, supported_mirror_versions: Dict[Tuple[str, Any], List[int]] + ) -> Tuple[bool, bool]: + items_to_remove = [] + clear, regenerate = False, not self._mirrors_for_spec + + for local_index_key in self._local_index_cache: + meta = MirrorMetadata.from_string(local_index_key) + if meta.version not in supported_mirror_versions.get((meta.url, meta.view), ()): + index_file_key = self._local_index_cache[local_index_key]["index_path"] + items_to_remove.append((local_index_key, index_file_key, meta)) + clear, regenerate = True, True + + for local_index_key, index_file_key, meta in items_to_remove: + self._last_fetch_times.pop(meta, None) + self._index_file_cache.remove(index_file_key) + del self._local_index_cache[local_index_key] + + return clear, regenerate def _fetch_and_cache_index(self, mirror_metadata: MirrorMetadata, cache_entry={}): """Fetch a buildcache index file from a remote mirror and cache it. @@ -483,7 +482,7 @@ def _fetch_and_cache_index(self, mirror_metadata: MirrorMetadata, cache_entry={} if not web_util.url_exists(index_url): raise BuildcacheIndexNotExists(f"Index not found in cache {index_url}") - fetcher: IndexFetcher = get_index_fetcher(scheme, mirror_metadata, cache_entry) + fetcher: IndexHandler = get_index_fetcher(scheme, mirror_metadata, cache_entry) result = fetcher.conditional_fetch() # Nothing to do @@ -493,7 +492,6 @@ def _fetch_and_cache_index(self, mirror_metadata: MirrorMetadata, cache_entry={} # Persist new index.json url_hash = compute_hash(str(mirror_metadata)) cache_key = "{}_{}.json".format(url_hash[:10], result.hash[:10]) - self._index_file_cache.init_entry(cache_key) with self._index_file_cache.write_transaction(cache_key) as (old, new): new.write(result.data) @@ -514,13 +512,13 @@ def _fetch_and_cache_index(self, mirror_metadata: MirrorMetadata, cache_entry={} def binary_index_location(): - """Set up a BinaryCacheIndex for remote buildcache dbs in the user's homedir.""" + """Set up a BinaryIndexCache for remote buildcache dbs in the user's homedir.""" cache_root = os.path.join(spack.caches.misc_cache_location(), "indices") return spack.util.path.canonicalize_path(cache_root) #: Default binary cache index instance -BINARY_INDEX: BinaryCacheIndex = spack.llnl.util.lang.Singleton(BinaryCacheIndex) # type: ignore +BINARY_INDEX = cast(BinaryIndexCache, spack.llnl.util.lang.Singleton(BinaryIndexCache)) def compute_hash(data): @@ -678,6 +676,8 @@ def _read_specs_and_push_index( cache_prefix: str, db: BuildCacheDatabase, temp_dir: str, + *, + timer=timer.NULL_TIMER, ): """Read listed specs, generate the index, and push it to the mirror. @@ -689,32 +689,34 @@ def _read_specs_and_push_index( db: A spack database used for adding specs and then writing the index. temp_dir: Location to write index.json and hash for pushing """ - for file in file_list: - # All supported versions of build caches put the hash as the last - # parameter before the extension - try: - x = file.split("/")[-1].split("-")[-1].split(".")[0] - except IndexError: - raise GenerateIndexError(f"Malformed metadata file name detected {file}") + with timer.measure("read"): + for file in file_list: + # All supported versions of build caches put the hash as the last + # parameter before the extension + try: + x = file.split("/")[-1].split("-")[-1].split(".")[0] + except IndexError: + raise GenerateIndexError(f"Malformed metadata file name detected {file}") - if not filter_fn(x): - continue + if not filter_fn(x): + continue - cache_entry: Optional[URLBuildcacheEntry] = None - try: - cache_entry = read_method(file) - spec_dict = cache_entry.fetch_metadata() - fetched_spec = spack.spec.Spec.from_dict(spec_dict) - except Exception as e: - tty.warn(f"Unable to fetch spec for manifest {file} due to: {e}") - continue - finally: - if cache_entry: - cache_entry.destroy() - db.add(fetched_spec) - db.mark(fetched_spec, "in_buildcache", True) + cache_entry: Optional[URLBuildcacheEntry] = None + try: + cache_entry = read_method(file) + spec_dict = cache_entry.fetch_metadata() + fetched_spec = spack.spec.Spec.from_dict(spec_dict) + except Exception as e: + tty.warn(f"Unable to fetch spec for manifest {file} due to: {e}") + continue + finally: + if cache_entry: + cache_entry.destroy() + db.add(fetched_spec) + db.mark(fetched_spec, "in_buildcache", True) - _push_index(db, temp_dir, cache_prefix, name) + with timer.measure("push"): + _push_index(db, temp_dir, cache_prefix, name) def _url_generate_package_index( @@ -723,6 +725,8 @@ def _url_generate_package_index( db: Optional[BuildCacheDatabase] = None, name: str = "", filter_fn: Callable[[str], bool] = lambda x: True, + *, + timer=timer.NULL_TIMER, ): """Create or replace the build cache index on the given mirror. The buildcache index contains an entry for each binary package under the @@ -736,9 +740,10 @@ def _url_generate_package_index( """ with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpspecsdir: try: - filename_to_mtime_mapping, read_fn = get_entries_from_cache( - url, tmpspecsdir, component_type=BuildcacheComponent.SPEC - ) + with timer.measure("list"): + filename_to_mtime_mapping, read_fn = get_entries_from_cache( + url, tmpspecsdir, component_type=BuildcacheComponent.SPEC + ) file_list = list(filename_to_mtime_mapping.keys()) except ListMirrorSpecsError as e: raise GenerateIndexError(f"Unable to generate package index: {e}") from e @@ -751,7 +756,14 @@ def _url_generate_package_index( try: _read_specs_and_push_index( - file_list, read_fn, name, filter_fn, url, db, str(db.database_directory) + file_list, + read_fn, + name, + filter_fn, + url, + db, + str(db.database_directory), + timer=timer, ) except Exception as e: raise GenerateIndexError( @@ -1708,7 +1720,7 @@ def download_tarball( spack.mirrors.mirror.MirrorCollection(binary=True).values() ) if not configured_mirrors: - tty.die("Please add a spack mirror to allow download of pre-compiled packages.") + raise NoConfiguredBinaryMirrors() # Note on try_first and try_next: # mirrors_for_spec mostly likely came from spack caching remote @@ -1753,18 +1765,18 @@ def fetch_url_to_mirror( # Fetch the manifest try: - response = spack.oci.opener.urlopen( + with spack.oci.opener.urlopen( urllib.request.Request( url=ref.manifest_url(), headers={"Accept": ", ".join(spack.oci.oci.manifest_content_type)}, ) - ) + ) as response: + manifest = json.load(response) except Exception: continue # Download the config = spec.json and the relevant tarball try: - manifest = json.load(response) spec_digest = spack.oci.image.Digest.from_string(manifest["config"]["digest"]) tarball_digest = spack.oci.image.Digest.from_string( manifest["layers"][-1]["digest"] @@ -2251,17 +2263,22 @@ def get_keys( if not mirror_collection: tty.die("Please add a spack mirror to allow " + "download of build caches.") + fingerprints = [] for mirror in mirror_collection.values(): if not mirror.signed: # Don't bother fetching keys for unsigned mirrors continue - for layout_version in mirror.supported_layout_versions: fetch_url = mirror.fetch_url if layout_version == 2: - _get_keys_v2(fetch_url, install, trust, force) + mirror_layout_fingerprints = _get_keys_v2(fetch_url, install, trust, force) else: - _get_keys(fetch_url, layout_version, install, trust, force) + mirror_layout_fingerprints = _get_keys( + fetch_url, layout_version, install, trust, force + ) + if mirror_layout_fingerprints: + fingerprints.extend(mirror_layout_fingerprints) + return fingerprints def _get_keys( @@ -2270,7 +2287,7 @@ def _get_keys( install: bool = False, trust: bool = False, force: bool = False, -) -> None: +) -> Optional[List[str]]: cache_class = get_url_buildcache_class(layout_version=layout_version) tty.debug("Finding public keys in {0}".format(url_util.format(mirror_url))) @@ -2287,12 +2304,13 @@ def _get_keys( except BuildcacheEntryError as e: tty.debug(f"Failed to fetch key index due to: {e}") index_entry.destroy() - return + return None with open(index_blob_path, encoding="utf-8") as fd: json_index = json.load(fd) index_entry.destroy() + saved_fingerprints = [] for fingerprint, _ in json_index["keys"].items(): key_manifest_url = url_util.join(keys_prefix, f"{fingerprint}.key.manifest.json") key_entry = cache_class(mirror_url, allow_unsigned=True) @@ -2309,16 +2327,17 @@ def _get_keys( if trust: spack.util.gpg.trust(key_blob_path) tty.debug(f"Added {fingerprint} to trusted keys.") + saved_fingerprints.append(fingerprint) else: tty.debug( - "Will not add this key to trusted keys." - "Use -t to install all downloaded keys" + "Will not add this key to trusted keys.Use -t to install all downloaded keys" ) key_entry.destroy() + return saved_fingerprints -def _get_keys_v2(mirror_url, install=False, trust=False, force=False): +def _get_keys_v2(mirror_url, install=False, trust=False, force=False) -> Optional[List[str]]: cache_class = get_url_buildcache_class(layout_version=2) keys_url = url_util.join( @@ -2329,8 +2348,7 @@ def _get_keys_v2(mirror_url, install=False, trust=False, force=False): tty.debug("Finding public keys in {0}".format(url_util.format(mirror_url))) try: - _, _, json_file = web_util.read_from_url(keys_index) - json_index = sjson.load(json_file) + json_index = web_util.read_json(keys_index) except (web_util.SpackWebError, OSError, ValueError) as url_err: # TODO: avoid repeated request if web_util.url_exists(keys_index): @@ -2339,8 +2357,9 @@ def _get_keys_v2(mirror_url, install=False, trust=False, force=False): f" caught exception attempting to read from {url_util.format(keys_index)}." ) tty.error(url_err) - return + return None + saved_fingerprints = [] for fingerprint, key_attributes in json_index["keys"].items(): link = os.path.join(keys_url, fingerprint + ".pub") @@ -2358,11 +2377,12 @@ def _get_keys_v2(mirror_url, install=False, trust=False, force=False): if trust: spack.util.gpg.trust(stage.save_filename) tty.debug("Added this key to trusted keys.") + saved_fingerprints.append(fingerprint) else: tty.debug( - "Will not add this key to trusted keys." - "Use -t to install all downloaded keys" + "Will not add this key to trusted keys.Use -t to install all downloaded keys" ) + return saved_fingerprints def _url_push_keys( @@ -2556,7 +2576,7 @@ class BuildcacheIndexNotExists(Exception): FetchIndexResult = collections.namedtuple("FetchIndexResult", "etag hash data fresh") -class IndexFetcher: +class IndexHandler: def conditional_fetch(self) -> FetchIndexResult: raise NotImplementedError(f"{self.__class__.__name__} is abstract") @@ -2602,11 +2622,11 @@ def fetch_index_blob( return (computed_hash, blob_result) -class DefaultIndexFetcherV2(IndexFetcher): +class DefaultIndexHandlerV2(IndexHandler): """Fetcher for index.json, using separate index.json.hash as cache invalidation strategy""" - def __init__(self, url, local_hash, urlopen=web_util.urlopen): - self.url = url + def __init__(self, mirror_metadata, local_hash, urlopen=web_util.urlopen): + self.url = mirror_metadata.url self.local_hash = local_hash self.urlopen = urlopen self.headers = {"User-Agent": web_util.SPACK_USER_AGENT} @@ -2615,8 +2635,10 @@ def get_remote_hash(self): # Failure to fetch index.json.hash is not fatal url_index_hash = url_util.join(self.url, "build_cache", "index.json.hash") try: - response = self.urlopen(urllib.request.Request(url_index_hash, headers=self.headers)) - remote_hash = response.read(64) + with self.urlopen( + urllib.request.Request(url_index_hash, headers=self.headers) + ) as response: + remote_hash = response.read(64) except OSError: return None @@ -2641,10 +2663,20 @@ def conditional_fetch(self) -> FetchIndexResult: except OSError as e: raise FetchIndexError(f"Could not fetch index from {url_index}", e) from e - try: - result = io.TextIOWrapper(response, encoding="utf-8").read() - except (ValueError, OSError) as e: - raise FetchIndexError(f"Remote index {url_index} is invalid") from e + with response: + try: + result = io.TextIOWrapper(response, encoding="utf-8").read() + except (ValueError, OSError) as e: + raise FetchIndexError(f"Remote index {url_index} is invalid") from e + + # For now we only handle etags on http(s), since 304 error handling + # in s3:// is not there yet. + if urllib.parse.urlparse(self.url).scheme not in ("http", "https"): + etag = None + else: + etag = web_util.parse_etag( + response.headers.get("Etag", None) or response.headers.get("etag", None) + ) computed_hash = compute_hash(result) @@ -2656,25 +2688,16 @@ def conditional_fetch(self) -> FetchIndexResult: # wrong, as it's more of an issue with race conditions in the cache # invalidation strategy. - # For now we only handle etags on http(s), since 304 error handling - # in s3:// is not there yet. - if urllib.parse.urlparse(self.url).scheme not in ("http", "https"): - etag = None - else: - etag = web_util.parse_etag( - response.headers.get("Etag", None) or response.headers.get("etag", None) - ) - warn_v2_layout(self.url, "Fetching an index") return FetchIndexResult(etag=etag, hash=computed_hash, data=result, fresh=False) -class EtagIndexFetcherV2(IndexFetcher): +class EtagIndexHandlerV2(IndexHandler): """Fetcher for index.json, using ETags headers as cache invalidation strategy""" - def __init__(self, url, etag, urlopen=web_util.urlopen): - self.url = url + def __init__(self, mirror_metadata, etag, urlopen=web_util.urlopen): + self.url = mirror_metadata.url self.etag = etag self.urlopen = urlopen @@ -2693,15 +2716,18 @@ def conditional_fetch(self) -> FetchIndexResult: except OSError as e: # URLError, socket.timeout, etc. raise FetchIndexError(f"Could not fetch index {url}", e) from e - try: - result = io.TextIOWrapper(response, encoding="utf-8").read() - except (ValueError, OSError) as e: - raise FetchIndexError(f"Remote index {url} is invalid", e) from e + with response: + try: + result = io.TextIOWrapper(response, encoding="utf-8").read() + except (ValueError, OSError) as e: + raise FetchIndexError(f"Remote index {url} is invalid", e) from e - warn_v2_layout(self.url, "Fetching an index") + warn_v2_layout(self.url, "Fetching an index") + + etag_header_value = response.headers.get("Etag", None) or response.headers.get( + "etag", None + ) - headers = response.headers - etag_header_value = headers.get("Etag", None) or headers.get("etag", None) return FetchIndexResult( etag=web_util.parse_etag(etag_header_value), hash=compute_hash(result), @@ -2710,7 +2736,7 @@ def conditional_fetch(self) -> FetchIndexResult: ) -class OCIIndexFetcher(IndexFetcher): +class OCIIndexHandler(IndexHandler): def __init__(self, mirror_metadata: MirrorMetadata, local_hash, urlopen=None) -> None: self.local_hash = local_hash self.ref = spack.oci.image.ImageReference.from_url(mirror_metadata.url) @@ -2729,10 +2755,11 @@ def conditional_fetch(self) -> FetchIndexResult: except OSError as e: raise FetchIndexError(f"Could not fetch manifest from {url_manifest}", e) from e - try: - manifest = json.load(response) - except Exception as e: - raise FetchIndexError(f"Remote index {url_manifest} is invalid", e) from e + with response: + try: + manifest = json.load(response) + except Exception as e: + raise FetchIndexError(f"Remote index {url_manifest} is invalid", e) from e # Get first blob hash, which should be the index.json try: @@ -2746,13 +2773,13 @@ def conditional_fetch(self) -> FetchIndexResult: # Otherwise fetch the blob / index.json try: - response = self.urlopen( + with self.urlopen( urllib.request.Request( url=self.ref.blob_url(index_digest), headers={"Accept": "application/vnd.oci.image.layer.v1.tar+gzip"}, ) - ) - result = io.TextIOWrapper(response, encoding="utf-8").read() + ) as response: + result = io.TextIOWrapper(response, encoding="utf-8").read() except (OSError, ValueError) as e: raise FetchIndexError(f"Remote index {url_manifest} is invalid", e) from e @@ -2763,7 +2790,7 @@ def conditional_fetch(self) -> FetchIndexResult: return FetchIndexResult(etag=None, hash=index_digest.digest, data=result, fresh=False) -class DefaultIndexFetcher(IndexFetcher): +class DefaultIndexHandler(IndexHandler): """Fetcher for buildcache index, cache invalidation via manifest contents""" def __init__(self, mirror_metadata: MirrorMetadata, local_hash, urlopen=web_util.urlopen): @@ -2787,7 +2814,8 @@ def conditional_fetch(self) -> FetchIndexResult: f"Could not read index manifest from {url_index_manifest}" ) from e - index_blob_record = self.get_index_manifest(response) + with response: + index_blob_record = self.get_index_manifest(response) # Early exit if our cache is up to date. if self.local_hash and self.local_hash == index_blob_record.checksum: @@ -2810,10 +2838,10 @@ def conditional_fetch(self) -> FetchIndexResult: return FetchIndexResult(etag=etag, hash=computed_hash, data=result, fresh=False) -class EtagIndexFetcher(IndexFetcher): +class EtagIndexHandler(IndexHandler): """Fetcher for buildcache index, cache invalidation via ETags headers - This class differs from the :class:`DefaultIndexFetcher` in the following ways: + This class differs from the :class:`DefaultIndexHandler` in the following ways: 1. It is provided with an etag value on creation, rather than an index checksum value. Note that since we never start out with an etag, the default fetcher must have been used initially @@ -2850,15 +2878,16 @@ def conditional_fetch(self) -> FetchIndexResult: raise FetchIndexError(f"Could not fetch index manifest {manifest_url}", e) from e # We need to read the index manifest and fetch the associated blob + with response: + index_blob_record = self.get_index_manifest(response) + etag_header_value = response.headers.get("Etag", None) or response.headers.get( + "etag", None + ) + cache_entry = cache_class(self.url, allow_unsigned=True) - computed_hash, result = self.fetch_index_blob( - cache_entry, self.get_index_manifest(response) - ) + computed_hash, result = self.fetch_index_blob(cache_entry, index_blob_record) cache_entry.destroy() - headers = response.headers - etag_header_value = headers.get("Etag", None) or headers.get("etag", None) - return FetchIndexResult( etag=web_util.parse_etag(etag_header_value), hash=computed_hash, @@ -2869,23 +2898,23 @@ def conditional_fetch(self) -> FetchIndexResult: def get_index_fetcher( scheme: str, mirror_metadata: MirrorMetadata, cache_entry: Dict[str, str] -) -> IndexFetcher: +) -> IndexHandler: if scheme == "oci": # TODO: Actually etag and OCI are not mutually exclusive... - return OCIIndexFetcher(mirror_metadata, cache_entry.get("index_hash", None)) + return OCIIndexHandler(mirror_metadata, cache_entry.get("index_hash", None)) elif cache_entry.get("etag"): if mirror_metadata.version < 3: - return EtagIndexFetcherV2(mirror_metadata.url, cache_entry["etag"]) + return EtagIndexHandlerV2(mirror_metadata, cache_entry["etag"]) else: - return EtagIndexFetcher(mirror_metadata, cache_entry["etag"]) + return EtagIndexHandler(mirror_metadata, cache_entry["etag"]) else: if mirror_metadata.version < 3: - return DefaultIndexFetcherV2( - mirror_metadata.url, local_hash=cache_entry.get("index_hash", None) + return DefaultIndexHandlerV2( + mirror_metadata, local_hash=cache_entry.get("index_hash", None) ) else: - return DefaultIndexFetcher( + return DefaultIndexHandler( mirror_metadata, local_hash=cache_entry.get("index_hash", None) ) @@ -2952,3 +2981,10 @@ class CannotListKeys(GenerateIndexError): class PushToBuildCacheError(spack.error.SpackError): """Raised when unable to push objects to binary mirror""" + + +class NoConfiguredBinaryMirrors(spack.error.SpackError): + """Raised when no binary mirrors are configured but an operation requires them""" + + def __init__(self): + super().__init__("Please add a spack mirror to allow download of pre-compiled packages.") diff --git a/lib/spack/spack/bootstrap/_common.py b/lib/spack/spack/bootstrap/_common.py index 8531f92df058db..f1b45364e9d5fa 100644 --- a/lib/spack/spack/bootstrap/_common.py +++ b/lib/spack/spack/bootstrap/_common.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Common basic functions used through the spack.bootstrap package""" + import fnmatch import glob import importlib diff --git a/lib/spack/spack/bootstrap/clingo.py b/lib/spack/spack/bootstrap/clingo.py index eac32e18656e82..130dcd5b7f7424 100644 --- a/lib/spack/spack/bootstrap/clingo.py +++ b/lib/spack/spack/bootstrap/clingo.py @@ -9,6 +9,7 @@ This module contains the logic to get a concrete spec for clingo, starting from a prototype JSON file for a similar platform. """ + import pathlib import sys from typing import Dict, Optional, Tuple, Type diff --git a/lib/spack/spack/bootstrap/config.py b/lib/spack/spack/bootstrap/config.py index ef048e24d23022..09fec22acea7d5 100644 --- a/lib/spack/spack/bootstrap/config.py +++ b/lib/spack/spack/bootstrap/config.py @@ -147,9 +147,7 @@ def _ensure_bootstrap_configuration() -> Generator: ), spack.config.use_configuration( # Default configuration scopes excluding command line and builtin *_bootstrap_config_scopes() - ), spack.store.use_store( - bootstrap_store_path, extra_data={"padded_length": 0} - ): + ), spack.store.use_store(bootstrap_store_path, extra_data={"padded_length": 0}): spack.config.set("bootstrap", user_configuration["bootstrap"]) spack.config.set("config", user_configuration["config"]) spack.config.set("repos", user_configuration["repos"]) diff --git a/lib/spack/spack/bootstrap/core.py b/lib/spack/spack/bootstrap/core.py index 025088256acdb8..9bf8419cac4523 100644 --- a/lib/spack/spack/bootstrap/core.py +++ b/lib/spack/spack/bootstrap/core.py @@ -34,6 +34,7 @@ import spack.config import spack.detection import spack.error +import spack.installer_dispatch import spack.mirrors.mirror import spack.platforms import spack.spec @@ -44,7 +45,6 @@ import spack.util.spack_yaml import spack.util.url import spack.version -from spack.installer import PackageInstaller from spack.llnl.util import tty from spack.llnl.util.lang import GroupedExceptionHandler @@ -291,7 +291,7 @@ def try_import(self, module: str, abstract_spec_str: str) -> bool: # Install the spec that should make the module importable with spack.config.override(self.mirror_scope): - PackageInstaller( + spack.installer_dispatch.create_installer( [concrete_spec.package], fail_fast=True, root_policy="source_only", @@ -319,7 +319,7 @@ def try_search_path(self, executables: Tuple[str], abstract_spec_str: str) -> bo msg = "[BOOTSTRAP] Try installing '{0}' from sources" tty.debug(msg.format(abstract_spec_str)) with spack.config.override(self.mirror_scope): - PackageInstaller([concrete_spec.package], fail_fast=True).install() + spack.installer_dispatch.create_installer([concrete_spec.package]).install() if _executables_in_store(executables, concrete_spec, query_info=info): self.last_search = info return True diff --git a/lib/spack/spack/bootstrap/environment.py b/lib/spack/spack/bootstrap/environment.py index 144bd0bd7cf019..7fffbfc9239950 100644 --- a/lib/spack/spack/bootstrap/environment.py +++ b/lib/spack/spack/bootstrap/environment.py @@ -2,21 +2,26 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Bootstrap non-core Spack dependencies from an environment.""" + +import contextlib import hashlib import os import pathlib +import shutil import sys from typing import Iterable, List import spack.vendor.archspec.cpu +import spack.binary_distribution +import spack.config import spack.environment import spack.spec import spack.tengine +import spack.util.gpg import spack.util.path from spack.llnl.util import tty -from ._common import _root_spec from .config import root_path, spec_for_current_python, store_path from .core import _add_externals_if_missing @@ -32,18 +37,12 @@ def __init__(self) -> None: # Remove python package roots created before python-venv was introduced for s in self.concrete_roots(): if "python" in s.package.extendees and not s.dependencies("python-venv"): - self.deconcretize(s) + self.deconcretize_by_hash(s.dag_hash()) @classmethod def spack_dev_requirements(cls) -> List[str]: """Spack development requirements""" - return [ - isort_root_spec(), - mypy_root_spec(), - black_root_spec(), - flake8_root_spec(), - pytest_root_spec(), - ] + return [pytest_root_spec(), ruff_root_spec(), mypy_root_spec()] @classmethod def environment_root(cls) -> pathlib.Path: @@ -78,6 +77,12 @@ def spack_yaml(cls) -> pathlib.Path: """Environment spack.yaml file""" return cls.environment_root().joinpath("spack.yaml") + @contextlib.contextmanager + def trust_bootstrap_mirror_keys(self): + with spack.util.gpg.gnupghome_override(os.path.join(root_path(), ".bootstrap-gpg")): + spack.binary_distribution.get_keys(install=True, trust=True) + yield + def update_installations(self) -> None: """Update the installations of this environment.""" log_enabled = tty.is_debug() or tty.is_verbose() @@ -91,8 +96,23 @@ def update_installations(self) -> None: tty.msg(f"[BOOTSTRAPPING] Installing dependencies ({', '.join(colorized_specs)})") self.write(regenerate=False) with tty.SuppressOutput(msg_enabled=log_enabled, warn_enabled=log_enabled): - self.install_all(fail_fast=True) - self.write(regenerate=True) + with self.trust_bootstrap_mirror_keys(): + fetch_policy = ( + "cache_only" + if not spack.config.get("bootstrap:dev:enable_source", False) + else "auto" + ) + try: + self.install_all( + fail_fast=True, + root_policy=fetch_policy, + dependencies_policy=fetch_policy, + ) + except BaseException: + # catch any exception as we always want to clean up + shutil.rmtree(self.environment_root()) + raise + self.write(regenerate=True) def load(self) -> None: """Update PATH and sys.path.""" @@ -110,39 +130,40 @@ def _write_spack_yaml_file(self) -> None: template = env.get_template("bootstrap/spack.yaml") context = { "python_spec": f"{spec_for_current_python()}+ctypes", - "python_prefix": sys.exec_prefix, + "python_prefix": pathlib.Path(sys.exec_prefix).as_posix(), "architecture": spack.vendor.archspec.cpu.host().family, - "environment_path": self.environment_root(), + "environment_path": self.environment_root().as_posix(), "environment_specs": self.spack_dev_requirements(), - "store_path": store_path(), + "store_path": pathlib.Path(store_path()).as_posix(), + "bootstrap_mirrors": dev_bootstrap_mirror_names(), } self.environment_root().mkdir(parents=True, exist_ok=True) self.spack_yaml().write_text(template.render(context), encoding="utf-8") -def isort_root_spec() -> str: - """Return the root spec used to bootstrap isort""" - return _root_spec("py-isort@5") - - def mypy_root_spec() -> str: """Return the root spec used to bootstrap mypy""" - return _root_spec("py-mypy@0.900: ^py-mypy-extensions@:1.0") + return "py-mypy@0.900: ^py-mypy-extensions@:1.0" -def black_root_spec() -> str: - """Return the root spec used to bootstrap black""" - return _root_spec("py-black@:25.1.0") +def pytest_root_spec() -> str: + """Return the root spec used to bootstrap pytest""" + return "py-pytest@6.2.4:" -def flake8_root_spec() -> str: - """Return the root spec used to bootstrap flake8""" - return _root_spec("py-flake8@3.8.2:") +def ruff_root_spec() -> str: + """Return the root spec used to bootstrap ruff""" + return "py-ruff@0.15.0" -def pytest_root_spec() -> str: - """Return the root spec used to bootstrap flake8""" - return _root_spec("py-pytest@6.2.4:") +def dev_bootstrap_mirror_names() -> List[str]: + """Return the mirror names used for bootstrapping dev + requirements""" + return [ + "developer-tools-darwin", + "developer-tools-x86_64_v3-linux-gnu", + "developer-tools-aarch64-linux-gnu", + ] def ensure_environment_dependencies() -> None: diff --git a/lib/spack/spack/bootstrap/status.py b/lib/spack/spack/bootstrap/status.py index 1626fe8115832d..4e1fbda20b7661 100644 --- a/lib/spack/spack/bootstrap/status.py +++ b/lib/spack/spack/bootstrap/status.py @@ -2,22 +2,16 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Query the status of bootstrapping on this machine""" + import sys -from typing import Dict, List, Optional, Sequence, Tuple, Union +from typing import List, Optional, Sequence, Tuple, Union import spack.util.executable from ._common import _executables_in_store, _python_import, _try_import_from_store from .config import ensure_bootstrap_configuration -from .core import clingo_root_spec, patchelf_root_spec -from .environment import ( - BootstrapEnvironment, - black_root_spec, - flake8_root_spec, - isort_root_spec, - mypy_root_spec, - pytest_root_spec, -) +from .core import clingo_root_spec, gnupg_root_spec, patchelf_root_spec +from .environment import BootstrapEnvironment, mypy_root_spec, pytest_root_spec, ruff_root_spec ExecutablesType = Union[str, Sequence[str]] RequiredResponseType = Tuple[bool, Optional[str]] @@ -85,15 +79,17 @@ def _core_requirements() -> List[RequiredResponseType]: def _buildcache_requirements() -> List[RequiredResponseType]: - _buildcache_exes: Dict[ExecutablesType, str] = { - ("gpg2", "gpg"): _missing("gpg2", "required to sign/verify buildcaches", False) - } - if sys.platform == "darwin": - _buildcache_exes["otool"] = _missing("otool", "required to relocate binaries") - - # Executables that are not bootstrapped yet - result = [_required_system_executable(exe, msg) for exe, msg in _buildcache_exes.items()] + # Add bootstrappable executables (these can be in PATH or bootstrapped) + # GPG/GPG2 - used for signing and verifying buildcaches + result = [ + _required_executable( + ("gpg2", "gpg"), + gnupg_root_spec(), + _missing("gpg2", "required to sign/verify buildcaches", False), + ) + ] + # Patchelf - only needed on Linux, used for binary relocation if sys.platform == "linux": result.append( _required_executable( @@ -124,20 +120,16 @@ def _development_requirements() -> List[RequiredResponseType]: env.load() return [ - _required_executable( - "isort", isort_root_spec(), _missing("isort", "required for style checks", False) - ), - _required_executable( - "mypy", mypy_root_spec(), _missing("mypy", "required for style checks", False) + _required_python_module( + "pytest", pytest_root_spec(), _missing("pytest", "required to run unit-test", False) ), _required_executable( - "flake8", flake8_root_spec(), _missing("flake8", "required for style checks", False) + "ruff", + ruff_root_spec(), + _missing("ruff", "required for code checking/formatting", False), ), _required_executable( - "black", black_root_spec(), _missing("black", "required for code formatting", False) - ), - _required_python_module( - "pytest", pytest_root_spec(), _missing("pytest", "required to run unit-test", False) + "mypy", mypy_root_spec(), _missing("mypy", "required for type checks", False) ), ] diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 24591e6a351913..84e1e9030cde2a 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -14,7 +14,7 @@ This is how things are set up when install() is called. Spack takes advantage of each package being in its own module by adding a bunch of command-like functions (like configure(), make(), etc.) in - the package's module scope. Ths allows package writers to call + the package's module scope. This allows package writers to call them all directly in Package.install() without writing 'self.' everywhere. No, this isn't Pythonic. Yes, it makes the code more readable and more like the shell script from which someone is @@ -48,13 +48,13 @@ from multiprocessing.connection import Connection from typing import ( Any, + BinaryIO, Callable, Dict, List, Optional, Sequence, Set, - TextIO, Tuple, Type, Union, @@ -79,6 +79,7 @@ import spack.store import spack.subprocess_context import spack.util.executable +import spack.util.module_cmd from spack import traverse from spack.context import Context from spack.error import InstallError, NoHeadersError, NoLibrariesError @@ -100,7 +101,6 @@ ) from spack.util.executable import Executable from spack.util.log_parse import make_log_context, parse_log_events -from spack.util.module_cmd import load_module # # This can be set by the user to globally disable parallel builds. @@ -199,9 +199,9 @@ def __call__( timeout: Optional[int] = ..., env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., - input: Optional[TextIO] = ..., - output: Union[Optional[TextIO], str] = ..., - error: Union[Optional[TextIO], str] = ..., + input: Optional[BinaryIO] = ..., + output: Union[Optional[BinaryIO], str] = ..., + error: Union[Optional[BinaryIO], str] = ..., _dump_env: Optional[Dict[str, str]] = ..., ) -> None: ... @@ -218,9 +218,9 @@ def __call__( timeout: Optional[int] = ..., env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., - input: Optional[TextIO] = ..., + input: Optional[BinaryIO] = ..., output: Union[Type[str], Callable] = ..., - error: Union[Optional[TextIO], str, Type[str], Callable] = ..., + error: spack.util.executable.OutType = ..., _dump_env: Optional[Dict[str, str]] = ..., ) -> str: ... @@ -237,8 +237,8 @@ def __call__( timeout: Optional[int] = ..., env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., - input: Optional[TextIO] = ..., - output: Union[Optional[TextIO], str, Type[str], Callable] = ..., + input: Optional[BinaryIO] = ..., + output: spack.util.executable.OutType = ..., error: Union[Type[str], Callable] = ..., _dump_env: Optional[Dict[str, str]] = ..., ) -> str: ... @@ -787,9 +787,9 @@ def setup_package(pkg, dirty, context: Context = Context.BUILD): tty.debug("setup_package: adding compiler wrappers paths") env_by_name = env_mods.group_by_name() for x in env_by_name["SPACK_COMPILER_WRAPPER_PATH"]: - assert isinstance( - x, PrependPath - ), "unexpected setting used for SPACK_COMPILER_WRAPPER_PATH" + assert isinstance(x, PrependPath), ( + "unexpected setting used for SPACK_COMPILER_WRAPPER_PATH" + ) env_mods.prepend_path("PATH", x.value) # Check whether we want to force RPATH or RUNPATH @@ -1117,7 +1117,7 @@ def load_external_modules(context: SetupContext) -> None: for spec, _ in context.external: external_modules = spec.external_modules or [] for external_module in external_modules: - load_module(external_module) + spack.util.module_cmd.load_module(external_module) def _setup_pkg_and_run( @@ -1654,7 +1654,7 @@ def _make_child_error(msg, module, name, traceback, log, log_type, context): def write_log_summary(out, log_type, log, last=None): - errors, warnings = parse_log_events(log) + errors, warnings, _ = parse_log_events(log) nerr = len(errors) nwar = len(warnings) diff --git a/lib/spack/spack/buildcache_migrate.py b/lib/spack/spack/buildcache_migrate.py index 0782bba17f4525..2b6e28d1ed4f0a 100644 --- a/lib/spack/spack/buildcache_migrate.py +++ b/lib/spack/spack/buildcache_migrate.py @@ -2,7 +2,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import io import json import os import pathlib @@ -104,8 +103,7 @@ def _migrate_spec( for meta_url in v2_metadata_urls: try: - _, _, meta_file = web_util.read_from_url(meta_url) - spec_contents = io.TextIOWrapper(meta_file, encoding="utf-8").read() + spec_contents = web_util.read_text(meta_url) v2_spec_url = meta_url break except (web_util.SpackWebError, OSError): @@ -279,8 +277,7 @@ def migrate( contents = None try: - _, _, index_file = web_util.read_from_url(index_url) - contents = io.TextIOWrapper(index_file, encoding="utf-8").read() + contents = web_util.read_text(index_url) except (web_util.SpackWebError, OSError): raise MigrationException("Buildcache migration requires a buildcache index") @@ -295,7 +292,7 @@ def migrate( specs_to_migrate = [ s for s in db.query_local(installed=InstallRecordStatus.ANY) - # todo, make it easer to get install records associated with specs + # todo, make it easier to get install records associated with specs if not s.external and db._data[s.dag_hash()].in_buildcache ] diff --git a/lib/spack/spack/buildcache_prune.py b/lib/spack/spack/buildcache_prune.py index b5d3a144e5176f..fab85ed02cb2ea 100644 --- a/lib/spack/spack/buildcache_prune.py +++ b/lib/spack/spack/buildcache_prune.py @@ -1,7 +1,6 @@ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import os import pathlib import re import tempfile @@ -18,16 +17,9 @@ import spack.util.parallel import spack.util.url as url_util import spack.util.web as web_util -from spack.util.executable import which from .mirrors.mirror import Mirror -from .url_buildcache import ( - CURRENT_BUILD_CACHE_LAYOUT_VERSION, - BuildcacheComponent, - URLBuildcacheEntry, - get_entries_from_cache, - get_url_buildcache_class, -) +from .url_buildcache import BuildcacheComponent, URLBuildcacheEntry, get_entries_from_cache def _fetch_manifests( @@ -60,65 +52,20 @@ def _fetch_manifests( ) for blob_name in blobs ] + tty.debug(f"Found {len(blobs)} blobs") return manifest_file_to_mtime_mapping, read_fn, blobs -def _delete_manifests_from_cache_aws( - url: str, tmpspecsdir: str, urls_to_delete: Set[str] -) -> Optional[int]: - aws = which("aws") - - if not aws: - tty.warn("AWS CLI not found, skipping deletion of cache entries.") - return None - - cache_class = get_url_buildcache_class(layout_version=CURRENT_BUILD_CACHE_LAYOUT_VERSION) - - include_pattern = cache_class.get_buildcache_component_include_pattern( - BuildcacheComponent.MANIFEST - ) - - file_count_before_deletion = len(list(pathlib.Path(tmpspecsdir).rglob(include_pattern))) - - tty.debug(f"Deleting {len(urls_to_delete)} entries from cache at {url}") - deleted = _delete_entries_from_cache_manual(tmpspecsdir, urls_to_delete) - tty.debug(f"Deleted {deleted} entries from cache at {url}") - - sync_command_args = [ - "s3", - "sync", - "--delete", - "--exclude", - "*", - "--include", - include_pattern, - tmpspecsdir, - url, - ] - - try: - aws(*sync_command_args, output=os.devnull, error=os.devnull) - # `aws s3 sync` doesn't return the number of deleted files, - # but we can calculate it based on the local file count from - # before and after the deletion. - return file_count_before_deletion - len( - list(pathlib.Path(tmpspecsdir).rglob(include_pattern)) - ) - except Exception: - tty.warn( - "Failed to use aws s3 sync to delete manifests, falling back to parallel deletion." - ) - - return None - - -def _delete_entries_from_cache_manual(url: str, urls_to_delete: Set[str]) -> int: +def _delete_entries_from_cache( + manifests_to_delete: Set[str], blobs_to_delete: Set[str], dry_run: bool +) -> int: + urls_to_delete = blobs_to_delete.union(manifests_to_delete) pruned_objects = 0 futures: List[Future] = [] with spack.util.parallel.make_concurrent_executor() as executor: for url in urls_to_delete: - futures.append(executor.submit(_delete_object, url)) + futures.append(executor.submit(_delete_object, url, dry_run)) for manifest_or_blob_future in as_completed(futures): pruned_objects += manifest_or_blob_future.result() @@ -126,36 +73,13 @@ def _delete_entries_from_cache_manual(url: str, urls_to_delete: Set[str]) -> int return pruned_objects -def _delete_entries_from_cache( - mirror: Mirror, tmpspecsdir: str, manifests_to_delete: Set[str], blobs_to_delete: Set[str] -) -> int: - pruned_manifests: Optional[int] = None - - if mirror.fetch_url.startswith("s3://"): - pruned_manifests = _delete_manifests_from_cache_aws( - url=mirror.fetch_url, tmpspecsdir=tmpspecsdir, urls_to_delete=manifests_to_delete - ) - - if pruned_manifests is None: - # If the AWS CLI deletion failed, we fall back to deleting both manifests - # and blobs with the fallback method. - objects_to_delete = blobs_to_delete.union(manifests_to_delete) - pruned_objects = 0 - else: - # If the AWS CLI deletion succeeded, we only need to worry about - # deleting the blobs, since the manifests have already been deleted. - objects_to_delete = blobs_to_delete - pruned_objects = pruned_manifests - - return pruned_objects + _delete_entries_from_cache_manual( - url=mirror.fetch_url, urls_to_delete=objects_to_delete - ) - - -def _delete_object(url: str) -> int: +def _delete_object(url: str, dry_run: bool) -> int: try: - web_util.remove_url(url=url) - tty.info(f"Removed object {url}") + if dry_run: + tty.info(f"Would have removed object {url}") + else: + web_util.remove_url(url=url) + tty.info(f"Removed object {url}") return 1 except Exception as e: tty.warn(f"Unable to remove object {url} due to: {e}") @@ -171,7 +95,7 @@ def _object_has_prunable_mtime(url: str, pruning_started_at: float) -> Tuple[str stat_result = web_util.stat_url(url) assert stat_result is not None if stat_result[1] > pruning_started_at: - tty.verbose(f"Skipping deletion of {url} because it was modified after pruning started") + tty.info(f"Skipping deletion of {url} because it was modified after pruning started") return url, False return url, True @@ -275,24 +199,8 @@ def _prune_orphans( if orphaned_manifests: tty.info(f"Found {len(orphaned_manifests)} manifest(s) that are missing blobs") - # If dry run, just print the manifests and blobs that would be deleted - # and exit early. - if dry_run: - pruned_object_count = len(orphaned_blobs) + len(orphaned_manifests) - for manifest in orphaned_manifests: - manifests.remove(manifest) - tty.info(f" Would prune manifest: {manifest}") - for blob in orphaned_blobs: - blobs.remove(blob) - tty.info(f" Would prune blob: {blob}") - return pruned_object_count - - # Otherwise, perform the deletions. pruned_object_count = _delete_entries_from_cache( - mirror=mirror, - tmpspecsdir=tmpspecsdir, - manifests_to_delete=orphaned_manifests, - blobs_to_delete=orphaned_blobs, + manifests_to_delete=orphaned_manifests, blobs_to_delete=orphaned_blobs, dry_run=dry_run ) for manifest in orphaned_manifests: @@ -304,7 +212,14 @@ def _prune_orphans( def prune_direct( - mirror: Mirror, keeplist_file: pathlib.Path, pruning_started_at: float, dry_run: bool + mirror: Mirror, + keeplist_file: pathlib.Path, + manifest_to_mtime_mapping: Dict[str, float], + read_fn: Callable[[str], URLBuildcacheEntry], + blob_list: List[str], + tmpspecsdir: str, + pruning_started_at: float, + dry_run: bool, ) -> None: """ Execute direct pruning for a given mirror using a keeplist file. @@ -335,67 +250,62 @@ def prune_direct( tty.info(f"Loaded {len(keep_hashes)} hashes to keep from {keeplist_file}") total_pruned: Optional[int] = None - with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpspecsdir: - try: - manifest_to_mtime_mapping, read_fn, blob_list = _fetch_manifests(mirror, tmpspecsdir) - except Exception as e: - raise BuildcachePruningException("Error getting entries from buildcache") from e + manifests_url = url_util.join( + mirror.fetch_url, + *URLBuildcacheEntry.get_relative_path_components(BuildcacheComponent.MANIFEST), + ) - # Determine which manifests correspond to specs we want to prune - manifests_to_prune: List[str] = [] - specs_to_prune: List[str] = [] + # Determine which manifests correspond to specs we want to prune + manifests_to_prune: List[str] = [] + specs_to_prune: List[str] = [] - for manifest in manifest_to_mtime_mapping.keys(): - if not fnmatch( - manifest, - URLBuildcacheEntry.get_buildcache_component_include_pattern( - BuildcacheComponent.SPEC - ), - ): - tty.info(f"Found a non-spec manifest at {manifest}, skipping...") - continue + tty.info(f"Found {len(manifest_to_mtime_mapping)} total manifests in mirror") - # Attempt to regex match the manifest name in order to extract the name, version, - # and hash for the spec. - manifest_name = manifest.split("/")[-1] # strip off parent directories - regex_match = re.match(r"([^ ]+)-([^- ]+)[-_]([^-_\. ]+)", manifest_name) + for manifest in manifest_to_mtime_mapping.keys(): + # Convert back from local to remote path. + manifest = manifest.replace(tmpspecsdir, manifests_url) + if not fnmatch( + manifest, + URLBuildcacheEntry.get_buildcache_component_include_pattern(BuildcacheComponent.SPEC), + ): + tty.debug(f"Found a non-spec manifest at {manifest}, skipping...") + continue - if regex_match is None: - # This should never happen, unless the buildcache is somehow corrupted - # and/or there is a bug. - raise BuildcachePruningException( - "Unable to extract spec name, version, and hash from " - f'the manifest named "{manifest_name}"' - ) + # Attempt to regex match the manifest name in order to extract the name, version, + # and hash for the spec. + manifest_name = manifest.split("/")[-1] # strip off parent directories + # Schema is -- + regex_match = re.match(r"([^ ]+)-([^- ]+)[-_]([^-_\. ]+)", manifest_name) + + if regex_match is None: + # This should never happen, unless the buildcache is somehow corrupted + # and/or there is a bug. + raise BuildcachePruningException( + "Unable to extract spec name, version, and hash from " + f'the manifest named "{manifest_name}"' + ) - spec_name, spec_version, spec_hash = regex_match.groups() + spec_name, spec_version, spec_hash = regex_match.groups() - # Chop off any prefix/parent file path to get just the name - spec_name = pathlib.Path(spec_name).name + if spec_hash not in keep_hashes: + manifests_to_prune.append(manifest) + specs_to_prune.append(f"{spec_name}/{spec_hash[:7]}") - if spec_hash not in keep_hashes: - manifests_to_prune.append(manifest) - specs_to_prune.append(f"{spec_name}/{spec_hash[:7]}") + if not manifests_to_prune: + tty.info("No specs to prune - all specs are in the keeplist") + return - if not manifests_to_prune: - tty.info("No specs to prune - all specs are in the keeplist") - return + manifests_to_delete = set(_filter_new_specs(manifests_to_prune, pruning_started_at)) - tty.info(f"Found {len(manifests_to_prune)} spec(s) to prune") + tty.info(f"Found {len(manifests_to_delete)} spec(s) to prune") - if dry_run: - for spec_name in specs_to_prune: - tty.info(f" Would prune: {spec_name}") - total_pruned = len(manifests_to_prune) - else: - manifests_to_delete = set(_filter_new_specs(manifests_to_prune, pruning_started_at)) + total_pruned = _delete_entries_from_cache( + manifests_to_delete=manifests_to_delete, blobs_to_delete=set(), dry_run=dry_run + ) - total_pruned = _delete_entries_from_cache( - mirror=mirror, - tmpspecsdir=tmpspecsdir, - manifests_to_delete=manifests_to_delete, - blobs_to_delete=set(), - ) + # Remove pruned specs from manifest_to_mtime_mapping. + for manifest in manifests_to_delete: + manifest_to_mtime_mapping.pop(manifest, None) if dry_run: tty.info(f"Would have pruned {total_pruned} objects from mirror: {mirror.fetch_url}") @@ -408,7 +318,15 @@ def prune_direct( tty.info("Run `spack buildcache update-index` to update the index for this mirror.") -def prune_orphan(mirror: Mirror, pruning_started_at: float, dry_run: bool) -> None: +def prune_orphan( + mirror: Mirror, + manifest_to_mtime_mapping: Dict[str, float], + read_fn: Callable[[str], URLBuildcacheEntry], + blob_list: List[str], + tmpspecsdir: str, + pruning_started_at: float, + dry_run: bool, +) -> None: """ Execute the pruning process for a given mirror. @@ -418,43 +336,36 @@ def prune_orphan(mirror: Mirror, pruning_started_at: float, dry_run: bool) -> No tty.debug(f"Pruning mirror: {mirror.fetch_url}" + (" (dry run)" if dry_run else "")) total_pruned = 0 - with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpspecsdir: - try: - manifest_to_mtime_mapping, read_fn, blob_list = _fetch_manifests(mirror, tmpspecsdir) - manifests = list(manifest_to_mtime_mapping.keys()) - except Exception as e: - raise BuildcachePruningException("Error getting entries from buildcache") from e - while True: - # Continue pruning until no more orphaned objects are found - pruned = _prune_orphans( - mirror=mirror, - manifests=manifests, - read_fn=read_fn, - blobs=blob_list, - pruning_started_at=pruning_started_at, - tmpspecsdir=tmpspecsdir, - dry_run=dry_run, - ) - if pruned == 0: - break - total_pruned += pruned + manifests = list(manifest_to_mtime_mapping.keys()) + + while True: + # Continue pruning until no more orphaned objects are found + pruned = _prune_orphans( + mirror=mirror, + manifests=manifests, + read_fn=read_fn, + blobs=blob_list, + pruning_started_at=pruning_started_at, + tmpspecsdir=tmpspecsdir, + dry_run=dry_run, + ) + if pruned == 0: + break + total_pruned += pruned - if dry_run: + if dry_run: + tty.info( + f"Would have pruned {total_pruned} orphaned objects from mirror: " + mirror.fetch_url + ) + else: + tty.info(f"Pruned {total_pruned} orphaned objects from mirror: {mirror.fetch_url}") + if total_pruned > 0: + # If we pruned any objects, the buildcache index is likely out of date. + # Inform the user about this. tty.info( - f"Would have pruned {total_pruned} orphaned objects from mirror: " - + mirror.fetch_url + "As a consequence of pruning, the buildcache index is now likely out of date." ) - else: - tty.info(f"Pruned {total_pruned} orphaned objects from mirror: {mirror.fetch_url}") - if total_pruned > 0: - # If we pruned any objects, the buildcache index is likely out of date. - # Inform the user about this. - tty.info( - "As a consequence of pruning, the buildcache index is now likely out of date." - ) - tty.info( - "Run `spack buildcache update-index` to update the index for this mirror." - ) + tty.info("Run `spack buildcache update-index` to update the index for this mirror.") def get_buildcache_normalized_time(mirror: Mirror) -> float: @@ -463,7 +374,7 @@ def get_buildcache_normalized_time(mirror: Mirror) -> float: This is necessary because different buildcache implementations may use different time formats/time zones. This function creates a temporary file, calls `stat_url` - on it, and then deletes it. This guarentees that the time used for the beginning + on it, and then deletes it. This guarantees that the time used for the beginning of the pruning is consistent across all buildcache implementations. """ with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as f: @@ -504,10 +415,27 @@ def prune_buildcache(mirror: Mirror, keeplist: Optional[str] = None, dry_run: bo else: started_at = get_buildcache_normalized_time(mirror) - if keeplist: - prune_direct(mirror, pathlib.Path(keeplist), started_at, dry_run) + with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpspecsdir: + try: + manifest_to_mtime_mapping, read_fn, blob_list = _fetch_manifests(mirror, tmpspecsdir) + except Exception as e: + raise BuildcachePruningException("Error getting entries from buildcache") from e + + if keeplist: + prune_direct( + mirror, + pathlib.Path(keeplist), + manifest_to_mtime_mapping, + read_fn, + blob_list, + tmpspecsdir, + started_at, + dry_run, + ) - prune_orphan(mirror, started_at, dry_run) + prune_orphan( + mirror, manifest_to_mtime_mapping, read_fn, blob_list, tmpspecsdir, started_at, dry_run + ) class BuildcachePruningException(spack.error.SpackError): diff --git a/lib/spack/spack/builder.py b/lib/spack/spack/builder.py index 1a6bcc6feeb70c..35a3564be4209f 100644 --- a/lib/spack/spack/builder.py +++ b/lib/spack/spack/builder.py @@ -70,9 +70,12 @@ def __call__(self, spec, prefix): def get_builder_class(pkg, name: str) -> Optional[Type["Builder"]]: """Return the builder class if a package module defines it.""" - cls = getattr(pkg.module, name, None) - if cls and spack.repo.is_package_module(cls.__module__): - return cls + for current_cls in type(pkg).__mro__: + if not hasattr(current_cls, "module"): + continue + maybe_builder = getattr(current_cls.module, name, None) + if maybe_builder and spack.repo.is_package_module(maybe_builder.__module__): + return maybe_builder return None diff --git a/lib/spack/spack/caches.py b/lib/spack/spack/caches.py index a01189cf2f0248..528969295f6673 100644 --- a/lib/spack/spack/caches.py +++ b/lib/spack/spack/caches.py @@ -3,7 +3,8 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Caches used by Spack to store data""" -import os + +from typing import cast import spack.config import spack.fetch_strategy @@ -11,7 +12,6 @@ import spack.paths import spack.util.file_cache import spack.util.path -from spack.llnl.util.filesystem import mkdirp def misc_cache_location(): @@ -30,9 +30,7 @@ def _misc_cache(): #: Spack's cache for small data -MISC_CACHE: spack.util.file_cache.FileCache = spack.llnl.util.lang.Singleton( # type: ignore - _misc_cache -) +MISC_CACHE = cast(spack.util.file_cache.FileCache, spack.llnl.util.lang.Singleton(_misc_cache)) def fetch_cache_location(): @@ -53,21 +51,18 @@ def _fetch_cache(): return spack.fetch_strategy.FsCache(path) -class MirrorCache: +class MirrorCache(spack.fetch_strategy.FsCacheBase): def __init__(self, root, skip_unstable_versions): - self.root = os.path.abspath(root) + super().__init__(root) self.skip_unstable_versions = skip_unstable_versions def store(self, fetcher, relative_dest): - """Fetch and relocate the fetcher's target into our mirror cache.""" + """Fetch and relocate the fetcher's target into our mirror cache. - # Note this will archive package sources even if they would not - # normally be cached (e.g. the current tip of an hg/git branch) - dst = os.path.join(self.root, relative_dest) - mkdirp(os.path.dirname(dst)) - fetcher.archive(dst) + Note: archives package sources even if not normally cached (e.g. tip of hg/git branch). + """ + super().store(fetcher, relative_dest) #: Spack's local cache for downloaded source archives -FETCH_CACHE: "spack.fetch_strategy.FsCache" -FETCH_CACHE = spack.llnl.util.lang.Singleton(_fetch_cache) # type: ignore +FETCH_CACHE = cast(spack.fetch_strategy.FsCache, spack.llnl.util.lang.Singleton(_fetch_cache)) diff --git a/lib/spack/spack/ci/__init__.py b/lib/spack/spack/ci/__init__.py index 8906e9aa6d966c..2e900bc6ed7969 100644 --- a/lib/spack/spack/ci/__init__.py +++ b/lib/spack/spack/ci/__init__.py @@ -3,7 +3,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import base64 -import io import json import os import pathlib @@ -495,13 +494,7 @@ def generate_pipeline(env: ev.Environment, args) -> None: rebuild_everything = not options.prune_up_to_date and not options.prune_untouched # Build a pipeline from the specs in the concrete environment - pipeline = PipelineDag( - [ - concrete - for abstract, concrete in env.concretized_specs() - if abstract in env.spec_lists["specs"] - ] - ) + pipeline = PipelineDag([env.specs_by_hash[x.hash] for x in env.concretized_roots]) # Optionally add various pruning filters pruning_filters = [] @@ -721,9 +714,9 @@ def download_and_extract_artifacts(url: str, work_dir: str) -> str: os.makedirs(work_dir, exist_ok=True) try: - response = urlopen(request, timeout=SPACK_CDASH_TIMEOUT) - with open(artifacts_zip_path, "wb") as out_file: - shutil.copyfileobj(response, out_file) + with urlopen(request, timeout=SPACK_CDASH_TIMEOUT) as response: + with open(artifacts_zip_path, "wb") as out_file: + shutil.copyfileobj(response, out_file) with zipfile.ZipFile(artifacts_zip_path) as zip_file: zip_file.extractall(work_dir) @@ -995,7 +988,7 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime, use_local_head) # Regular expressions for parsing that HEAD commit. If the pipeline # was on the gitlab spack mirror, it will have been a merge commit made by - # gitub and pushed by the sync script. If the pipeline was run on some + # github and pushed by the sync script. If the pipeline was run on some # environment repo, then the tested spack commit will likely have been # a regular commit. commit_1 = None @@ -1136,7 +1129,7 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime, use_local_head) process_command("reproducer", entrypoint_script, work_dir, run=autostart) inst_list.append("\nOnce on the tagged runner:\n\n") - inst_list.extent( + inst_list.extend( [ " - Run the reproducer script", f" $ {work_dir}/reproducer.{platform_script_ext}", @@ -1305,12 +1298,11 @@ def read_broken_spec(broken_spec_url): object. """ try: - _, _, fs = web_util.read_from_url(broken_spec_url) + broken_spec_contents = web_util.read_text(broken_spec_url) except web_util.SpackWebError: tty.warn(f"Unable to read broken spec from {broken_spec_url}") return None - broken_spec_contents = io.TextIOWrapper(fs, encoding="utf-8").read() return syaml.load(broken_spec_contents) diff --git a/lib/spack/spack/ci/common.py b/lib/spack/spack/ci/common.py index 26b75f59b47988..581ce99b5704a0 100644 --- a/lib/spack/spack/ci/common.py +++ b/lib/spack/spack/ci/common.py @@ -92,7 +92,6 @@ def copy_files_to_artifacts( compress_artifacts (bool): option to compress copied artifacts using Gzip """ try: - if compress_artifacts: copy_gzipped(src, artifacts_dir) else: @@ -268,7 +267,8 @@ def create_buildgroup(self): group_id = None try: - response_text = _urlopen(request, timeout=SPACK_CDASH_TIMEOUT).read() + with _urlopen(request, timeout=SPACK_CDASH_TIMEOUT) as response: + response_text = response.read() except OSError as e: tty.warn(f"Failed to create CDash buildgroup: {e}") @@ -543,7 +543,7 @@ def __job_name(name, suffix=""): return jname def __apply_submapping(self, dest, spec, section): - """Apply submapping setion to the IR dict""" + """Apply submapping section to the IR dict""" matched = False only_first = section.get("match_behavior", "first") == "first" @@ -581,6 +581,7 @@ def generate_ir(self): "script": [ "cd {env_dir}", "spack env activate --without-view .", + "spack spec /$SPACK_JOB_SPEC_DAG_HASH", "spack ci rebuild", ] } @@ -593,7 +594,7 @@ def generate_ir(self): # Reindex script { "reindex-job": { - "script:": ["spack buildcache update-index --keys {index_target_mirror}"] + "script:": ["spack -v buildcache update-index --keys {index_target_mirror}"] } }, # Cleanup script @@ -713,8 +714,8 @@ def job_query(job): endpoint_url._replace(query=query).geturl(), headers=header, method="GET" ) try: - response = _urlopen(request) - config = json.load(response) + with _urlopen(request) as response: + config = json.load(response) except Exception as e: # For now just ignore any errors from dynamic mapping and continue # This is still experimental, and failures should not stop CI diff --git a/lib/spack/spack/ci/generator_registry.py b/lib/spack/spack/ci/generator_registry.py index f77b257e162e9d..70572f4c83a2ec 100644 --- a/lib/spack/spack/ci/generator_registry.py +++ b/lib/spack/spack/ci/generator_registry.py @@ -5,6 +5,7 @@ """Generators that support writing out pipelines for various CI platforms, using a common pipeline graph definition. """ + import spack.error _generators = {} diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 3f621e902b66b6..92f97e7b9b7e69 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -182,7 +182,8 @@ def parse_specs( args = [args] if isinstance(args, str) else args arg_string = " ".join([quote_kvp(arg) for arg in args]) - specs = spack.spec_parser.parse(arg_string) + toolchains = spack.config.CONFIG.get("toolchains", {}) + specs = spack.spec_parser.parse(arg_string, toolchains=toolchains) if not concretize: return specs @@ -485,7 +486,8 @@ def get_arg(name, default=None): if flags: ffmt += " {compiler_flags}" vfmt = "{variants}" if variants else "" - format_string = nfmt + "{@version}" + vfmt + ffmt + hfmt = "{/abstract_hash}" + format_string = nfmt + "{@version}" + vfmt + ffmt + hfmt if specfile_format: format_string = "[{specfile_version}] " + format_string @@ -637,29 +639,26 @@ def extant_file(f): return f -def require_active_env(cmd_name): - """Used by commands to get the active environment +def require_active_env(parser): + """Used by commands to get the active environment. - If an environment is not found, print an error message that says the calling - command *needs* an active environment. + If an environment is not found, calls ``parser.error()`` which prints usage and exits. Arguments: - cmd_name (str): name of calling command + parser: the subparser for the command (typically ``args.subparser``) Returns: (spack.environment.Environment): the active environment """ env = ev.active_environment() - if env: return env - - tty.die( - "`spack %s` requires an environment" % cmd_name, - "activate an environment first:", - " spack env activate ENV", - "or use:", - " spack -e ENV %s ..." % cmd_name, + parser.error( + "requires an active environment\n" + " activate an environment first:\n" + " spack env activate ENV\n" + " or use:\n" + " spack -e ENV %s ..." % parser.prog.partition(" ")[2] ) @@ -760,7 +759,7 @@ def group_arguments( prefix_length: length of any additional arguments (including spaces) to be passed before the groups from args; default is 0 characters max_group_length: max length of characters that if a group of args is joined by ``" "`` - On unix, ths defaults to SC_ARG_MAX from sysconf. On Windows the default is + On unix, this defaults to SC_ARG_MAX from sysconf. On Windows the default is the max usable for CreateProcess (32,768 chars) """ @@ -770,7 +769,7 @@ def group_arguments( max_group_length = 32766 if hasattr(os, "sysconf"): # sysconf is only on unix try: - # returns -1 if an option isn't present (soem older POSIXes) + # returns -1 if an option isn't present (some older POSIXes) sysconf_max = os.sysconf("SC_ARG_MAX") max_group_length = sysconf_max if sysconf_max != -1 else max_group_length except (ValueError, OSError): diff --git a/lib/spack/spack/cmd/add.py b/lib/spack/spack/cmd/add.py index 05ac69b65a5ae7..30ffa4a163be36 100644 --- a/lib/spack/spack/cmd/add.py +++ b/lib/spack/spack/cmd/add.py @@ -25,7 +25,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: def add(parser, args): - env = spack.cmd.require_active_env(cmd_name="add") + env = spack.cmd.require_active_env(args.subparser) with env.write_transaction(): for spec in spack.cmd.parse_specs(args.specs): diff --git a/lib/spack/spack/cmd/audit.py b/lib/spack/spack/cmd/audit.py index cd7359af2670e0..d9d0d29c16e710 100644 --- a/lib/spack/spack/cmd/audit.py +++ b/lib/spack/spack/cmd/audit.py @@ -68,7 +68,7 @@ def packages(parser, args): def packages_https(parser, args): # Since packages takes a long time, --all is required without name if not args.check_all and not args.name: - tty.die("Please specify one or more packages to audit, or --all.") + args.subparser.error("please specify one or more packages to audit, or --all") pkgs = args.name or spack.repo.PATH.all_package_names() reports = spack.audit.run_group(args.subcommand, pkgs=pkgs) diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index 0de13d0d2fa2c1..a5b9dda0f99a6e 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -270,9 +270,8 @@ def _write_bootstrapping_source_status(name, enabled, scope=None): matches = [s for s in sources if s["name"] == name] if not matches: names = [s["name"] for s in sources] - msg = ( - 'there is no bootstrapping method named "{0}". Valid ' - "method names are: {1}".format(name, ", ".join(names)) + msg = 'there is no bootstrapping method named "{0}". Valid method names are: {1}'.format( + name, ", ".join(names) ) raise RuntimeError(msg) @@ -381,8 +380,7 @@ def _remove(args): sources = [s for s in sources if s["name"] != args.name] spack.config.set("bootstrap:sources", sources, scope=current_scope) msg = ( - 'Removed the bootstrapping source named "{0}" from the ' - '"{1}" configuration scope.' + 'Removed the bootstrapping source named "{0}" from the "{1}" configuration scope.' ) spack.llnl.util.tty.msg(msg.format(args.name, current_scope)) trusted = spack.config.get("bootstrap:trusted", scope=current_scope) or [] diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py index f102806b6948f0..80fb3250dd2de3 100644 --- a/lib/spack/spack/cmd/buildcache.py +++ b/lib/spack/spack/cmd/buildcache.py @@ -25,6 +25,7 @@ import spack.stage import spack.store import spack.util.parallel +import spack.util.timer as timer_mod import spack.util.web as web_util from spack import traverse from spack.binary_distribution import BINARY_INDEX @@ -118,6 +119,12 @@ def setup_parser(subparser: argparse.ArgumentParser): action="store_true", help="stop pushing on first failure (default is best effort)", ) + push.add_argument( + "--allow-missing", + action="store_true", + help="allow not installed specs to continue without failure (default fails on missing " + "specs)", + ) push.add_argument( "--base-image", default=None, help="specify the base image for the buildcache" ) @@ -133,8 +140,17 @@ def setup_parser(subparser: argparse.ArgumentParser): action="store_true", help="for a private mirror, include non-redistributable packages", ) + push.add_argument( + "--group", + action="append", + default=None, + dest="groups", + metavar="GROUP", + help="push only specs from the given environment group " + "(can be specified multiple times, requires an active environment)", + ) arguments.add_common_arguments(push, ["specs", "jobs"]) - push.set_defaults(func=push_fn) + push.set_defaults(func=push_fn, subparser=push) install = subparsers.add_parser("install", help=install_fn.__doc__) install.add_argument( @@ -157,7 +173,7 @@ def setup_parser(subparser: argparse.ArgumentParser): ) arguments.add_common_arguments(install, ["specs"]) - install.set_defaults(func=install_fn) + install.set_defaults(func=install_fn, subparser=install) listcache = subparsers.add_parser("list", help=list_fn.__doc__) arguments.add_common_arguments(listcache, ["long", "very_long", "namespaces"]) @@ -175,7 +191,7 @@ def setup_parser(subparser: argparse.ArgumentParser): help="list specs for all available architectures instead of default platform and OS", ) arguments.add_common_arguments(listcache, ["specs"]) - listcache.set_defaults(func=list_fn) + listcache.set_defaults(func=list_fn, subparser=listcache) keys = subparsers.add_parser("keys", help=keys_fn.__doc__) keys.add_argument( @@ -183,7 +199,7 @@ def setup_parser(subparser: argparse.ArgumentParser): ) keys.add_argument("-t", "--trust", action="store_true", help="trust all downloaded keys") keys.add_argument("-f", "--force", action="store_true", help="force new download of keys") - keys.set_defaults(func=keys_fn) + keys.set_defaults(func=keys_fn, subparser=keys) # Check if binaries need to be rebuilt on remote mirror check = subparsers.add_parser("check", help=check_fn.__doc__) @@ -209,7 +225,7 @@ def setup_parser(subparser: argparse.ArgumentParser): arguments.add_common_arguments(check, ["specs"]) - check.set_defaults(func=check_fn) + check.set_defaults(func=check_fn, subparser=check) # Download tarball and specfile download = subparsers.add_parser("download", help=download_fn.__doc__) @@ -221,7 +237,7 @@ def setup_parser(subparser: argparse.ArgumentParser): default=None, help="path to directory where tarball should be downloaded", ) - download.set_defaults(func=download_fn) + download.set_defaults(func=download_fn, subparser=download) prune = subparsers.add_parser("prune", help=prune_fn.__doc__) prune.add_argument( @@ -238,7 +254,7 @@ def setup_parser(subparser: argparse.ArgumentParser): action="store_true", help="do not actually delete anything from the buildcache, but log what would be deleted", ) - prune.set_defaults(func=prune_fn) + prune.set_defaults(func=prune_fn, subparser=prune) # Given the root spec, save the yaml of the dependent spec to a file savespecfile = subparsers.add_parser("save-specfile", help=save_specfile_fn.__doc__) @@ -253,7 +269,7 @@ def setup_parser(subparser: argparse.ArgumentParser): savespecfile.add_argument( "--specfile-dir", required=True, help="path to directory where spec yamls should be saved" ) - savespecfile.set_defaults(func=save_specfile_fn) + savespecfile.set_defaults(func=save_specfile_fn, subparser=savespecfile) # Sync buildcache entries from one mirror to another sync = subparsers.add_parser("sync", help=sync_fn.__doc__) @@ -288,7 +304,7 @@ def setup_parser(subparser: argparse.ArgumentParser): help="destination mirror name, path, or URL", ) - sync.set_defaults(func=sync_fn) + sync.set_defaults(func=sync_fn, subparser=sync) # Check the validity of a buildcache check_index = subparsers.add_parser("check-index", help=check_index_fn.__doc__) @@ -308,7 +324,7 @@ def setup_parser(subparser: argparse.ArgumentParser): check_index.add_argument( "mirror", type=arguments.mirror_name_or_url, help="mirror name, path, or URL" ) - check_index.set_defaults(func=check_index_fn) + check_index.set_defaults(func=check_index_fn, subparser=check_index) # Update buildcache index without copying any additional packages update_index = subparsers.add_parser( @@ -332,7 +348,7 @@ def setup_parser(subparser: argparse.ArgumentParser): "-a", action="store_true", help="Append the listed specs to the current view index if it already exists. " - "This operation does not guarentee atomic write and should be run with care.", + "This operation does not guarantee atomic write and should be run with care.", ) update_index_view_mode_args.add_argument( "--force", @@ -349,7 +365,7 @@ def setup_parser(subparser: argparse.ArgumentParser): help="if provided, key index will be updated as well as package index", ) arguments.add_common_arguments(update_index, ["yes_to_all"]) - update_index.set_defaults(func=update_index_fn) + update_index.set_defaults(func=update_index_fn, subparser=update_index) # Migrate a buildcache from layout_version 2 to version 3 migrate = subparsers.add_parser("migrate", help=migrate_fn.__doc__) @@ -370,7 +386,7 @@ def setup_parser(subparser: argparse.ArgumentParser): ) arguments.add_common_arguments(migrate, ["yes_to_all"]) # TODO: add -y argument to prompt if user really means to delete existing - migrate.set_defaults(func=migrate_fn) + migrate.set_defaults(func=migrate_fn, subparser=migrate) def _matching_specs(specs: List[Spec]) -> List[Spec]: @@ -445,10 +461,23 @@ def _specs_to_be_packaged( def push_fn(args): """create a binary package and push it to a mirror""" - if args.specs: + if args.specs and args.groups: + args.subparser.error("--group and explicit specs are mutually exclusive") + + if args.groups: + env = spack.cmd.require_active_env(args.subparser) + available_groups = env.manifest.groups() + if any(g not in available_groups for g in args.groups): + tty.die( + f"Some of the groups do not exist in the environment. " + f"Available groups are: {', '.join(sorted(available_groups))}" + ) + + roots = [c for g in args.groups for _, c in env.concretized_specs_by(group=g)] + elif args.specs: roots = _matching_specs(spack.cmd.parse_specs(args.specs)) else: - roots = spack.cmd.require_active_env(cmd_name="buildcache push").concrete_roots() + roots = spack.cmd.require_active_env(args.subparser).concrete_roots() mirror = args.mirror assert isinstance(mirror, spack.mirrors.mirror.Mirror) @@ -492,8 +521,14 @@ def push_fn(args): with spack.store.STORE.db.read_transaction(): if any(not s.installed for s in specs): specs, not_installed = stable_partition(specs, lambda s: s.installed) - if args.fail_fast: + if args.fail_fast and not args.allow_missing: raise PackagesAreNotInstalledError(not_installed) + elif args.allow_missing: + tty.warn( + f"The following {len(not_installed)} specs are not installed and will be " + "skipped: \n" + + "\n".join(elide_list([f" {_format_spec(s)}" for s in not_installed], 5)) + ) else: failed.extend( (s, PackageNotInstalledError("package not installed")) for s in not_installed @@ -554,7 +589,7 @@ def push_fn(args): def install_fn(args): """install from a binary package""" if not args.specs: - tty.die("a spec argument is required to install from a buildcache") + args.subparser.error("a spec argument is required to install from a buildcache") query = spack.binary_distribution.BinaryCacheQuery(all_architectures=args.otherarch) matches = spack.store.find(args.specs, multiple=args.multiple, query_fn=query) @@ -605,7 +640,7 @@ def check_fn(args: argparse.Namespace): if specs_arg: specs = _matching_specs(spack.cmd.parse_specs(specs_arg)) else: - specs = spack.cmd.require_active_env("buildcache check").all_specs() + specs = spack.cmd.require_active_env(args.subparser).all_specs() if not specs: tty.msg("No specs provided, exiting.") @@ -642,7 +677,7 @@ def download_fn(args): specs = _matching_specs(spack.cmd.parse_specs(args.spec)) if len(specs) != 1: - tty.die("a single spec argument is required to download from a buildcache") + args.subparser.error("requires a single spec argument") spack.binary_distribution.download_single_spec(specs[0], args.path) @@ -657,7 +692,7 @@ def save_specfile_fn(args): specs = spack.cmd.parse_specs(args.root_spec) if len(specs) != 1: - tty.die("a single spec argument is required to save specfile") + args.subparser.error("requires a single spec argument") root = specs[0] @@ -757,13 +792,13 @@ def sync_fn(args): # specified, the second is ignored and the first is the override # destination. if args.dest_mirror: - tty.warn(f"Ignoring unused arguemnt: {args.dest_mirror.name}") + tty.warn(f"Ignoring unused argument: {args.dest_mirror.name}") manifest_copy(glob.glob(args.manifest_glob), args.src_mirror) return 0 if args.src_mirror is None or args.dest_mirror is None: - tty.die("Provide mirrors to sync from and to.") + args.subparser.error("provide mirrors to sync from and to") src_mirror = args.src_mirror dest_mirror = args.dest_mirror @@ -772,7 +807,7 @@ def sync_fn(args): dest_mirror_url = dest_mirror.push_url # Get the active environment - env = spack.cmd.require_active_env(cmd_name="buildcache sync") + env = spack.cmd.require_active_env(args.subparser) tty.msg( "Syncing environment buildcache files from {0} to {1}".format( @@ -796,7 +831,7 @@ def manifest_copy( manifest_file_list: List[str], dest_mirror: Optional[spack.mirrors.mirror.Mirror] = None ): """Read manifest files containing information about specific specs to copy - from source to destination, remove duplicates since any binary packge for + from source to destination, remove duplicates since any binary package for a given hash should be the same as any other, and copy all files specified in the manifest files.""" deduped_manifest = {} @@ -824,7 +859,10 @@ def manifest_copy( copy_buildcache_entry(src_cache_entry, destination_url) -def update_index(mirror: spack.mirrors.mirror.Mirror, update_keys=False): +def update_index( + mirror: spack.mirrors.mirror.Mirror, update_keys=False, timer=timer_mod.NULL_TIMER +): + timer.start() # Special case OCI images for now. try: image_ref = spack.oci.oci.image_from_mirror(mirror) @@ -842,7 +880,7 @@ def update_index(mirror: spack.mirrors.mirror.Mirror, update_keys=False): url = mirror.push_url with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir: - spack.binary_distribution._url_generate_package_index(url, tmpdir) + spack.binary_distribution._url_generate_package_index(url, tmpdir, timer=timer) if update_keys: mirror_update_keys(mirror) @@ -866,6 +904,7 @@ def update_view( name: Optional[str] = None, update_keys: bool = False, yes_to_all: bool = False, + parser, ): """update a buildcache view index""" # OCI images do not support views. @@ -925,7 +964,7 @@ def update_view( hashes.extend(env.all_hashes()) else: # Get hashes in the current active environment - hashes = spack.cmd.require_active_env(cmd_name="buildcache update-view").all_hashes() + hashes = spack.cmd.require_active_env(parser).all_hashes() if not hashes: tty.warn("No specs found for view, creating an empty index") @@ -942,7 +981,9 @@ def update_view( cache_index = BINARY_INDEX._local_index_cache.get(str(mirror_metadata)) if cache_index: cache_key = cache_index["index_path"] - db._read_from_file(BINARY_INDEX._index_file_cache.cache_path(cache_key)) + with BINARY_INDEX._index_file_cache.read_transaction(cache_key) as f: + if f is not None: + db._read_from_stream(f) spack.binary_distribution._url_generate_package_index(url, tmpdir, db, name, filter_fn) @@ -1002,9 +1043,9 @@ def check_index_fn(args): db = spack.binary_distribution.BuildCacheDatabase(tmpdir) cache_entry = BINARY_INDEX._local_index_cache[str(mirror_metadata)] cache_key = cache_entry["index_path"] - cache_path = BINARY_INDEX._index_file_cache.cache_path(cache_key) - with BINARY_INDEX._index_file_cache.read_transaction(cache_key): - db._read_from_file(cache_path) + with BINARY_INDEX._index_file_cache.read_transaction(cache_key) as f: + if f is not None: + db._read_from_stream(f) index_hash_list = set( [ @@ -1015,7 +1056,6 @@ def check_index_fn(args): ) for spec_manifest in manifest_files: - # Spec manifests have a naming format # --.spec.manifest.json spec_hash = spec_manifest.rsplit("-", 1)[1].split(".", 1)[0] @@ -1083,6 +1123,8 @@ def check_index_fn(args): def update_index_fn(args): """update a buildcache index or index view if extra arguments are provided.""" + t = timer_mod.Timer() if tty.is_verbose() else timer_mod.NullTimer() + update_view_index = ( args.append or args.force or args.name or args.sources or args.mirror.push_view ) @@ -1101,9 +1143,15 @@ def update_index_fn(args): name=args.name, update_keys=args.keys, yes_to_all=args.yes_to_all, + parser=args.subparser, ) else: - return update_index(args.mirror, update_keys=args.keys) + update_index(args.mirror, update_keys=args.keys, timer=t) + + if tty.is_verbose(): + tty.msg("Timing summary:") + t.stop() + t.write_tty() def migrate_fn(args): diff --git a/lib/spack/spack/cmd/change.py b/lib/spack/spack/cmd/change.py index 576eeb6aaa718f..33658f58f3a04f 100644 --- a/lib/spack/spack/cmd/change.py +++ b/lib/spack/spack/cmd/change.py @@ -57,11 +57,11 @@ def change(parser, args): if args.list_name != "specs" and args.concrete_only: warnings.warn("'spack change --list-name' argument is ignored with '--concrete-only'") - env = spack.cmd.require_active_env(cmd_name="change") + env = spack.cmd.require_active_env(args.subparser) match_spec = None if args.match_spec: - match_spec = spack.spec.Spec(args.match_spec) + match_spec = spack.cmd.parse_specs([args.match_spec])[0] specs = spack.cmd.parse_specs(args.specs) with env.write_transaction(): @@ -80,7 +80,12 @@ def change(parser, args): raise ValueError(msg) from e if args.concrete or args.concrete_only: + selectors = [] + mutators = [] for spec in specs: - env.mutate(selector=match_spec or spack.spec.Spec(spec.name), mutator=spec) + selectors.append(match_spec or spack.spec.Spec(spec.name)) + mutators.append(spec) + + env.mutate(selectors=selectors, mutators=mutators) env.write() diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py index 13ae4509793e36..76265a1025b91c 100644 --- a/lib/spack/spack/cmd/ci.py +++ b/lib/spack/spack/cmd/ci.py @@ -140,7 +140,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: help="path to the root of the artifacts directory\n\n" "The spack ci module assumes it will normally be run from within your project " "directory, wherever that is checked out to run your ci. The artifacts root directory " - "should specifiy a name that can safely be used for artifacts within your project " + "should specify a name that can safely be used for artifacts within your project " "directory.", ) generate.add_argument( @@ -149,7 +149,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: help="Environment variables to forward from the generate environment " "to the generated jobs.", ) - generate.set_defaults(func=ci_generate) + generate.set_defaults(func=ci_generate, subparser=generate) spack.cmd.common.arguments.add_concretizer_args(generate) spack.cmd.common.arguments.add_common_arguments(generate, ["jobs"]) @@ -159,7 +159,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: index = subparsers.add_parser( "rebuild-index", description=doc_dedented(ci_reindex), help=doc_first_line(ci_reindex) ) - index.set_defaults(func=ci_reindex) + index.set_defaults(func=ci_reindex, subparser=index) # Handle steps of a ci build/rebuild rebuild = subparsers.add_parser( @@ -192,7 +192,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: default=None, help="maximum time (in seconds) that tests are allowed to run", ) - rebuild.set_defaults(func=ci_rebuild) + rebuild.set_defaults(func=ci_rebuild, subparser=rebuild) spack.cmd.common.arguments.add_common_arguments(rebuild, ["jobs"]) # Facilitate reproduction of a failed CI build job @@ -231,7 +231,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: "--gpg-url", help="URL to public GPG key for validating binary cache installs" ) - reproduce.set_defaults(func=ci_reproduce) + reproduce.set_defaults(func=ci_reproduce, subparser=reproduce) # Verify checksums inside of ci workflows verify_versions = subparsers.add_parser( @@ -241,7 +241,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: ) verify_versions.add_argument("from_ref", help="git ref from which start looking at changes") verify_versions.add_argument("to_ref", help="git ref to end looking at changes") - verify_versions.set_defaults(func=ci_verify_versions) + verify_versions.set_defaults(func=ci_verify_versions, subparser=verify_versions) def ci_generate(args): @@ -252,7 +252,7 @@ def ci_generate(args): before invoking this command. the value must be the CDash authorization token needed to create a build group and register all generated jobs under it """ - env = spack.cmd.require_active_env(cmd_name="ci generate") + env = spack.cmd.require_active_env(args.subparser) spack_ci.generate_pipeline(env, args) @@ -263,7 +263,7 @@ def ci_reindex(args): use the active, gitlab-enabled environment to rebuild the buildcache index for the associated mirror """ - env = spack.cmd.require_active_env(cmd_name="ci rebuild-index") + env = spack.cmd.require_active_env(args.subparser) yaml_root = env.manifest[ev.TOP_LEVEL_KEY] if "mirrors" not in yaml_root or len(yaml_root["mirrors"].values()) < 1: @@ -286,7 +286,7 @@ def ci_rebuild(args): """ rebuild_timer = timer.Timer() - env = spack.cmd.require_active_env(cmd_name="ci rebuild") + env = spack.cmd.require_active_env(args.subparser) # Make sure the environment is "gitlab-enabled", or else there's nothing # to do. @@ -464,7 +464,7 @@ def ci_rebuild(args): # No hash match anywhere means we need to rebuild spec # Start with spack arguments - spack_cmd = [SPACK_COMMAND, "--color=always", "--backtrace", "--verbose", "install"] + spack_cmd = [SPACK_COMMAND, "--color=always", "install"] config = cfg.get("config") if not config["verify_ssl"]: @@ -490,7 +490,7 @@ def ci_rebuild(args): # Arguments when installing the root from sources deps_install_args = install_args + ["--only=dependencies"] - root_install_args = install_args + ["--keep-stage", "--only=package"] + root_install_args = install_args + ["--verbose", "--keep-stage", "--only=package"] if cdash_handler: # Add additional arguments to `spack install` for CDash reporting. @@ -740,9 +740,9 @@ def validate_standard_versions( for version in versions: url = pkg.find_valid_url_for_version(version) - assert ( - url is not None - ), f"Package {pkg.name} does not have a valid URL for version {version}" + assert url is not None, ( + f"Package {pkg.name} does not have a valid URL for version {version}" + ) url_dict[version] = url version_hashes = spack.stage.get_checksums_for_versions( diff --git a/lib/spack/spack/cmd/commands.py b/lib/spack/spack/cmd/commands.py index e88fb359749f31..4ebf3fa2691381 100644 --- a/lib/spack/spack/cmd/commands.py +++ b/lib/spack/spack/cmd/commands.py @@ -19,7 +19,7 @@ import spack.platforms from spack.llnl.util.argparsewriter import ArgparseRstWriter, ArgparseWriter, Command from spack.llnl.util.tty.colify import colify -from spack.main import section_descriptions +from spack.main import SpackArgumentParser, section_descriptions description = "list available spack commands" section = "config" @@ -196,9 +196,7 @@ def format(self, cmd: Command) -> str: assert not (cmd.positionals and cmd.subcommands) # one or the other # We only care about the arguments/flags, not the help messages - positionals: Tuple[str, ...] = () - if cmd.positionals: - positionals, _, _, _ = zip(*cmd.positionals) + positionals = cmd.positionals or () optionals, _, _, _, _ = zip(*cmd.optionals) subcommands: Tuple[str, ...] = () if cmd.subcommands: @@ -237,12 +235,12 @@ def end_function(self, prog: str) -> str: return "}\n" def body( - self, positionals: Sequence[str], optionals: Sequence[str], subcommands: Sequence[str] + self, positionals: Sequence, optionals: Sequence[str], subcommands: Sequence[str] ) -> str: """Return the body of the function. Args: - positionals: List of positional arguments. + positionals: List of positional argument tuples (name, choices, nargs, help). optionals: List of optional arguments. subcommands: List of subcommand parsers. @@ -272,21 +270,31 @@ def body( {self.optionals(optionals)} """ - def positionals(self, positionals: Sequence[str]) -> str: + def positionals(self, positionals: Sequence) -> str: """Return the syntax for reporting positional arguments. Args: - positionals: List of positional arguments. + positionals: List of positional argument tuples (name, choices, nargs, help). Returns: Syntax for positional arguments. """ - # If match found, return function name - for positional in positionals: + for name, choices, nargs, help in positionals: + # Check for a predefined subroutine mapping for key, value in _positional_to_subroutine.items(): - if positional.startswith(key): + if name.startswith(key): return value + # Use choices if available + if choices is not None: + if isinstance(choices, dict): + choices = sorted(choices.keys()) + elif isinstance(choices, (set, frozenset)): + choices = sorted(choices) + else: + choices = sorted(choices) + return 'SPACK_COMPREPLY="{}"'.format(" ".join(str(c) for c in choices)) + # If no matches found, return empty list return 'SPACK_COMPREPLY=""' @@ -688,8 +696,7 @@ def subcommands(args: Namespace, out: IO) -> None: args: Command-line arguments. out: File object to write to. """ - parser = spack.main.make_argument_parser() - spack.main.add_all_commands(parser) + parser = get_all_spack_commands(out) writer = SubcommandWriter(parser.prog, out, args.aliases) writer.write(parser) @@ -735,8 +742,7 @@ def rst(args: Namespace, out: IO) -> None: out: File object to write to. """ # create a parser with all commands - parser = spack.main.make_argument_parser() - spack.main.add_all_commands(parser) + parser = get_all_spack_commands(out) # extract cross-refs of the form `_cmd-spack-:` from rst files documented_commands: Set[str] = set() @@ -774,6 +780,20 @@ def names(args: Namespace, out: IO) -> None: colify(commands, output=out) +def get_all_spack_commands(out: IO) -> SpackArgumentParser: + is_tty = hasattr(out, "isatty") and out.isatty() + # Argparse python 3.14 adds a default color argument that + # adds color control characters to argparse output + # that breaks expected output format from spack formatters + # when written to non tty IO + # If 3.14 and newer and not tty, disable color + parser = spack.main.make_argument_parser( + **({"color": False} if sys.version_info[:2] >= (3, 14) and not is_tty else {}) + ) + spack.main.add_all_commands(parser) + return parser + + @formatter def bash(args: Namespace, out: IO) -> None: """Bash tab-completion script. @@ -782,9 +802,7 @@ def bash(args: Namespace, out: IO) -> None: args: Command-line arguments. out: File object to write to. """ - parser = spack.main.make_argument_parser() - spack.main.add_all_commands(parser) - + parser = get_all_spack_commands(out) aliases_config = spack.config.get("config:aliases") if aliases_config: aliases = ";".join(f"{key}:{val}" for key, val in aliases_config.items()) @@ -796,9 +814,7 @@ def bash(args: Namespace, out: IO) -> None: @formatter def fish(args, out): - parser = spack.main.make_argument_parser() - spack.main.add_all_commands(parser) - + parser = get_all_spack_commands(out) writer = FishCompletionWriter(parser.prog, out, args.aliases) writer.write(parser) @@ -830,7 +846,7 @@ def _commands(parser: ArgumentParser, args: Namespace) -> None: # check header first so we don't open out files unnecessarily if args.header and not os.path.exists(args.header): - tty.die(f"No such file: '{args.header}'") + args.subparser.error(f"no such file: '{args.header}'") if args.update: tty.msg(f"Updating file: {args.update}") @@ -868,7 +884,7 @@ def commands(parser: ArgumentParser, args: Namespace) -> None: """ if args.update_completion: if args.format != "names" or any([args.aliases, args.update, args.header]): - tty.die("--update-completion can only be specified alone.") + args.subparser.error("--update-completion can only be specified alone") # this runs the command multiple times with different arguments update_completion(parser, args) diff --git a/lib/spack/spack/cmd/common/arguments.py b/lib/spack/spack/cmd/common/arguments.py index 300d979c34b972..e31de62a73bbbc 100644 --- a/lib/spack/spack/cmd/common/arguments.py +++ b/lib/spack/spack/cmd/common/arguments.py @@ -105,7 +105,7 @@ def __call__(self, parser, namespace, jobs, option_string): # Jobs is a single integer, type conversion is already applied # see https://docs.python.org/3/library/argparse.html#action-classes if jobs < 1: - msg = 'invalid value for argument "{0}" ' '[expected a positive integer, got "{1}"]' + msg = 'invalid value for argument "{0}" [expected a positive integer, got "{1}"]' raise ValueError(msg.format(option_string, jobs)) spack.config.set("config:build_jobs", jobs, scope="command_line") @@ -122,7 +122,7 @@ class SetConcurrentPackages(argparse.Action): def __call__(self, parser, namespace, concurrent_packages, option_string): if concurrent_packages < 1: - msg = 'invalid value for argument "{0}" ' '[expected a positive integer, got "{1}"]' + msg = 'invalid value for argument "{0}" [expected a positive integer, got "{1}"]' raise ValueError(msg.format(option_string, concurrent_packages)) spack.config.set("config:concurrent_packages", concurrent_packages, scope="command_line") @@ -513,10 +513,10 @@ def add_cdash_args(subparser, add_help): "defaults to spec of the package to operate on" ) cdash_help["site"] = ( - "site name that will be reported to CDash\n\n" "defaults to current system hostname" + "site name that will be reported to CDash\n\ndefaults to current system hostname" ) cdash_help["track"] = ( - "results will be reported to this group on CDash\n\n" "defaults to Experimental" + "results will be reported to this group on CDash\n\ndefaults to Experimental" ) cdash_help["buildstamp"] = ( "use custom buildstamp\n\n" diff --git a/lib/spack/spack/cmd/common/spec_strings.py b/lib/spack/spack/cmd/common/spec_strings.py new file mode 100644 index 00000000000000..ec9ef96acba6a7 --- /dev/null +++ b/lib/spack/spack/cmd/common/spec_strings.py @@ -0,0 +1,232 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import ast +import os +import re +import sys +import warnings +from typing import Callable, List, Optional + +import spack.llnl.util.tty as tty +import spack.util.spack_yaml +from spack.spec_parser import NAME, VERSION_LIST, SpecTokens +from spack.tokenize import Token, TokenBase, Tokenizer + +IS_PROBABLY_COMPILER = re.compile(r"%[a-zA-Z_][a-zA-Z0-9\-]") + + +class _LegacySpecTokens(TokenBase): + """Reconstructs the tokens for previous specs, so we can reuse code to rotate them""" + + # Dependency + START_EDGE_PROPERTIES = r"(?:\^\[)" + END_EDGE_PROPERTIES = r"(?:\])" + DEPENDENCY = r"(?:\^)" + # Version + VERSION_HASH_PAIR = SpecTokens.VERSION_HASH_PAIR.regex + GIT_VERSION = SpecTokens.GIT_VERSION.regex + VERSION = SpecTokens.VERSION.regex + # Variants + PROPAGATED_BOOL_VARIANT = SpecTokens.PROPAGATED_BOOL_VARIANT.regex + BOOL_VARIANT = SpecTokens.BOOL_VARIANT.regex + PROPAGATED_KEY_VALUE_PAIR = SpecTokens.PROPAGATED_KEY_VALUE_PAIR.regex + KEY_VALUE_PAIR = SpecTokens.KEY_VALUE_PAIR.regex + # Compilers + COMPILER_AND_VERSION = rf"(?:%\s*(?:{NAME})(?:[\s]*)@\s*(?:{VERSION_LIST}))" + COMPILER = rf"(?:%\s*(?:{NAME}))" + # FILENAME + FILENAME = SpecTokens.FILENAME.regex + # Package name + FULLY_QUALIFIED_PACKAGE_NAME = SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME.regex + UNQUALIFIED_PACKAGE_NAME = SpecTokens.UNQUALIFIED_PACKAGE_NAME.regex + # DAG hash + DAG_HASH = SpecTokens.DAG_HASH.regex + # White spaces + WS = SpecTokens.WS.regex + # Unexpected character(s) + UNEXPECTED = SpecTokens.UNEXPECTED.regex + + +def _spec_str_reorder_compiler(idx: int, blocks: List[List[Token]]) -> None: + # only move the compiler to the back if it exists and is not already at the end + if not 0 <= idx < len(blocks) - 1: + return + # if there's only whitespace after the compiler, don't move it + if all(token.kind == _LegacySpecTokens.WS for block in blocks[idx + 1 :] for token in block): + return + # rotate left and always add at least one WS token between compiler and previous token + compiler_block = blocks.pop(idx) + if compiler_block[0].kind != _LegacySpecTokens.WS: + compiler_block.insert(0, Token(_LegacySpecTokens.WS, " ")) + # delete the WS tokens from the new first block if it was at the very start, to prevent leading + # WS tokens. + while idx == 0 and blocks[0][0].kind == _LegacySpecTokens.WS: + blocks[0].pop(0) + blocks.append(compiler_block) + + +def _spec_str_format(spec_str: str) -> Optional[str]: + """Given any string, try to parse as spec string, and rotate the compiler token to the end + of each spec instance. Returns the formatted string if it was changed, otherwise None.""" + # We parse blocks of tokens that include leading whitespace, and move the compiler block to + # the end when we hit a dependency ^... or the end of a string. + # [@3.1][ +foo][ +bar][ %gcc@3.1][ +baz] + # [@3.1][ +foo][ +bar][ +baz][ %gcc@3.1] + + current_block: List[Token] = [] + blocks: List[List[Token]] = [] + compiler_block_idx = -1 + in_edge_attr = False + + legacy_tokenizer = Tokenizer(_LegacySpecTokens) + + for token in legacy_tokenizer.tokenize(spec_str): + if token.kind == _LegacySpecTokens.UNEXPECTED: + # parsing error, we cannot fix this string. + return None + elif token.kind in (_LegacySpecTokens.COMPILER, _LegacySpecTokens.COMPILER_AND_VERSION): + # multiple compilers are not supported in Spack v0.x, so early return + if compiler_block_idx != -1: + return None + current_block.append(token) + blocks.append(current_block) + current_block = [] + compiler_block_idx = len(blocks) - 1 + elif token.kind in ( + _LegacySpecTokens.START_EDGE_PROPERTIES, + _LegacySpecTokens.DEPENDENCY, + _LegacySpecTokens.UNQUALIFIED_PACKAGE_NAME, + _LegacySpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, + ): + _spec_str_reorder_compiler(compiler_block_idx, blocks) + compiler_block_idx = -1 + if token.kind == _LegacySpecTokens.START_EDGE_PROPERTIES: + in_edge_attr = True + current_block.append(token) + blocks.append(current_block) + current_block = [] + elif token.kind == _LegacySpecTokens.END_EDGE_PROPERTIES: + in_edge_attr = False + current_block.append(token) + blocks.append(current_block) + current_block = [] + elif in_edge_attr: + current_block.append(token) + elif token.kind in ( + _LegacySpecTokens.VERSION_HASH_PAIR, + _LegacySpecTokens.GIT_VERSION, + _LegacySpecTokens.VERSION, + _LegacySpecTokens.PROPAGATED_BOOL_VARIANT, + _LegacySpecTokens.BOOL_VARIANT, + _LegacySpecTokens.PROPAGATED_KEY_VALUE_PAIR, + _LegacySpecTokens.KEY_VALUE_PAIR, + _LegacySpecTokens.DAG_HASH, + ): + current_block.append(token) + blocks.append(current_block) + current_block = [] + elif token.kind == _LegacySpecTokens.WS: + current_block.append(token) + else: + raise ValueError(f"unexpected token {token}") + + if current_block: + blocks.append(current_block) + _spec_str_reorder_compiler(compiler_block_idx, blocks) + + new_spec_str = "".join(token.value for block in blocks for token in block) + return new_spec_str if spec_str != new_spec_str else None + + +SpecStrHandler = Callable[[str, int, int, str, str], None] + + +def _spec_str_default_handler(path: str, line: int, col: int, old: str, new: str): + """A SpecStrHandler that prints formatted spec strings and their locations.""" + print(f"{path}:{line}:{col}: `{old}` -> `{new}`") + + +def _spec_str_fix_handler(path: str, line: int, col: int, old: str, new: str): + """A SpecStrHandler that updates formatted spec strings in files.""" + with open(path, "r", encoding="utf-8") as f: + lines = f.readlines() + new_line = lines[line - 1].replace(old, new) + if new_line == lines[line - 1]: + tty.warn(f"{path}:{line}:{col}: could not apply fix: `{old}` -> `{new}`") + return + lines[line - 1] = new_line + print(f"{path}:{line}:{col}: fixed `{old}` -> `{new}`") + with open(path, "w", encoding="utf-8") as f: + f.writelines(lines) + + +def _spec_str_ast(path: str, tree: ast.AST, handler: SpecStrHandler) -> None: + """Walk the AST of a Python file and apply handler to formatted spec strings.""" + for node in ast.walk(tree): + if sys.version_info >= (3, 8): + if isinstance(node, ast.Constant) and isinstance(node.value, str): + current_str = node.value + else: + continue + elif isinstance(node, ast.Str): + current_str = node.s + else: + continue + if not IS_PROBABLY_COMPILER.search(current_str): + continue + new = _spec_str_format(current_str) + if new is not None: + handler(path, node.lineno, node.col_offset, current_str, new) + + +def _spec_str_json_and_yaml(path: str, data: dict, handler: SpecStrHandler) -> None: + """Walk a YAML or JSON data structure and apply handler to formatted spec strings.""" + queue = [data] + seen = set() + + while queue: + current = queue.pop(0) + if id(current) in seen: + continue + seen.add(id(current)) + if isinstance(current, dict): + queue.extend(current.values()) + queue.extend(current.keys()) + elif isinstance(current, list): + queue.extend(current) + elif isinstance(current, str) and IS_PROBABLY_COMPILER.search(current): + new = _spec_str_format(current) + if new is not None: + mark = getattr(current, "_start_mark", None) + if mark: + line, col = mark.line + 1, mark.column + 1 + else: + line, col = 0, 0 + handler(path, line, col, current, new) + + +def _check_spec_strings( + paths: List[str], handler: SpecStrHandler = _spec_str_default_handler +) -> None: + """Open Python, JSON and YAML files, and format their string literals that look like spec + strings. A handler is called for each formatting, which can be used to print or apply fixes.""" + for path in paths: + is_json_or_yaml = path.endswith(".json") or path.endswith(".yaml") or path.endswith(".yml") + is_python = path.endswith(".py") + if not is_json_or_yaml and not is_python: + continue + + try: + with open(path, "r", encoding="utf-8") as f: + # skip files that are likely too large to be user code or config + if os.fstat(f.fileno()).st_size > 1024 * 1024: + warnings.warn(f"skipping {path}: too large.") + continue + if is_json_or_yaml: + _spec_str_json_and_yaml(path, spack.util.spack_yaml.load_config(f), handler) + elif is_python: + _spec_str_ast(path, ast.parse(f.read()), handler) + except (OSError, spack.util.spack_yaml.SpackYAMLError, SyntaxError, ValueError): + warnings.warn(f"skipping {path}") + continue diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index abe59a95cc589a..d195b3212d9b94 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -180,11 +180,14 @@ def compiler_info(args): def compiler_list(args): compilers = _all_available_compilers(scope=args.scope, remote=args.remote) + if not sys.stdout.isatty(): + for c in sorted(compilers): # type: ignore + print(c.format("{name}@{version}")) + return + # If there are no compilers in any scope, and we're outputting to a tty, give a # hint to the user. if len(compilers) == 0: - if not sys.stdout.isatty(): - return msg = "No compilers available" if args.scope is None: msg += ". Run `spack compiler find` to autodetect compilers" diff --git a/lib/spack/spack/cmd/concretize.py b/lib/spack/spack/cmd/concretize.py index 1d8884f7408616..10cde4f7aa57c6 100644 --- a/lib/spack/spack/cmd/concretize.py +++ b/lib/spack/spack/cmd/concretize.py @@ -31,7 +31,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: def concretize(parser, args): - env = spack.cmd.require_active_env(cmd_name="concretize") + env = spack.cmd.require_active_env(args.subparser) if args.test == "all": tests = True diff --git a/lib/spack/spack/cmd/config.py b/lib/spack/spack/cmd/config.py index 04bddc186ae1f4..32d21a5cc8cbfb 100644 --- a/lib/spack/spack/cmd/config.py +++ b/lib/spack/spack/cmd/config.py @@ -46,6 +46,13 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: choices=spack.config.SECTION_SCHEMAS, ) get_parser.add_argument("--json", action="store_true", help="output configuration as JSON") + get_parser.add_argument( + "--group", + metavar="group", + default=None, + help="show configuration as seen by this environment spec group (requires active env)", + ) + get_parser.set_defaults(subparser=get_parser) blame_parser = sp.add_parser( "blame", help="print configuration annotated with source file:line" @@ -57,6 +64,13 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: metavar="section", choices=spack.config.SECTION_SCHEMAS, ) + blame_parser.add_argument( + "--group", + metavar="group", + default=None, + help="show configuration as seen by this environment spec group (requires active env)", + ) + blame_parser.set_defaults(subparser=blame_parser) edit_parser = sp.add_parser("edit", help="edit configuration file") edit_parser.add_argument( @@ -69,8 +83,10 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: edit_parser.add_argument( "--print-file", action="store_true", help="print the file name that would be edited" ) + edit_parser.set_defaults(subparser=edit_parser) - sp.add_parser("list", help="list configuration sections") + list_parser = sp.add_parser("list", help="list configuration sections") + list_parser.set_defaults(subparser=list_parser) scopes_parser = sp.add_parser( "scopes", help="list defined scopes in descending order of precedence" @@ -107,6 +123,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: nargs="?", choices=spack.config.SECTION_SCHEMAS, ) + scopes_parser.set_defaults(subparser=scopes_parser) add_parser = sp.add_parser("add", help="add configuration parameters") add_parser.add_argument( @@ -115,10 +132,12 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: help="colon-separated path to config that should be added, e.g. 'config:default:true'", ) add_parser.add_argument("-f", "--file", help="file from which to set all config values") + add_parser.set_defaults(subparser=add_parser) change_parser = sp.add_parser("change", help="swap variants etc. on specs in config") change_parser.add_argument("path", help="colon-separated path to config section with specs") change_parser.add_argument("--match-spec", help="only change constraints that match this") + change_parser.set_defaults(subparser=change_parser) prefer_upstream_parser = sp.add_parser( "prefer-upstream", help="set package preferences from upstream" @@ -130,13 +149,14 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: default=False, help="set packages preferences based on local installs, rather than upstream", ) + prefer_upstream_parser.set_defaults(subparser=prefer_upstream_parser) remove_parser = sp.add_parser("remove", aliases=["rm"], help="remove configuration parameters") remove_parser.add_argument( "path", - help="colon-separated path to config that should be removed," - " e.g. 'config:default:true'", + help="colon-separated path to config that should be removed, e.g. 'config:default:true'", ) + remove_parser.set_defaults(subparser=remove_parser) # Make the add parser available later setattr(setup_parser, "add_parser", add_parser) @@ -144,12 +164,14 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: update = sp.add_parser("update", help="update configuration files to the latest format") arguments.add_common_arguments(update, ["yes_to_all"]) update.add_argument("section", help="section to update") + update.set_defaults(subparser=update) revert = sp.add_parser( "revert", help="revert configuration files to their state before update" ) arguments.add_common_arguments(revert, ["yes_to_all"]) revert.add_argument("section", help="section to update") + revert.set_defaults(subparser=revert) def _get_scope_and_section(args): @@ -179,10 +201,27 @@ def _get_scope_and_section(args): def print_configuration(args, *, blame: bool) -> None: if args.scope and args.scope not in spack.config.existing_scope_names(): - tty.die(f"the argument --scope={args.scope} must refer to an existing scope.") + args.subparser.error(f"the argument --scope={args.scope} must refer to an existing scope") if args.scope and args.section is None: - tty.die(f"the argument --scope={args.scope} requires specifying a section.") + args.subparser.error(f"the argument --scope={args.scope} requires specifying a section") + + group = getattr(args, "group", None) + if group is not None: + env = ev.active_environment() + if env is None: + args.subparser.error("the argument --group requires an active environment") + return # parser.error exits, but help mypy understand this is unreachable + try: + with env.config_override_for_group(group=group): + _print_configuration_helper(args, blame=blame) + except ValueError as e: + tty.die(str(e)) + return + + _print_configuration_helper(args, blame=blame) + +def _print_configuration_helper(args, *, blame: bool) -> None: yaml = blame or not args.json if args.section is not None: @@ -255,12 +294,13 @@ def config_edit(args): # If we aren't editing a spack.yaml file, get config path from scope. scope, section = _get_scope_and_section(args) if not scope and not section: - tty.die("`spack config edit` requires a section argument or an active environment.") + args.subparser.error("requires a section argument or an active environment") config_file = spack.config.CONFIG.get_config_filename(scope, section) if args.print_file: print(config_file) else: + fs.mkdirp(os.path.dirname(config_file)) editor(config_file) @@ -292,7 +332,7 @@ def _config_scope_info(args, scope, active, included): result.append( section_path if section_path and os.path.exists(section_path) - else f"{scope.path}{os.sep}" + else f"{scope.path}{'' if os.path.isfile(scope.path) else os.sep}" ) else: result.append(" ") diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 65f675de0a416b..3e17b6af2783aa 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -173,10 +173,11 @@ class AutoreconfPackageTemplate(PackageTemplate): ) dependencies = """\ - depends_on("autoconf", type="build") - depends_on("automake", type="build") - depends_on("libtool", type="build") - depends_on("m4", type="build") + with default_args(type="build"): + depends_on("autoconf") + depends_on("automake") + depends_on("libtool") + depends_on("m4") # FIXME: Add additional dependencies if required. # depends_on("foo")""" @@ -316,7 +317,8 @@ class BazelPackageTemplate(PackageTemplate): dependencies = """\ # FIXME: Add additional dependencies if required. - depends_on("bazel", type="build")""" + with default_args(type="build"): + depends_on("bazel")""" body_def = """\ def install(self, spec, prefix): @@ -325,7 +327,7 @@ def install(self, spec, prefix): class RacketPackageTemplate(PackageTemplate): - """Provides approriate overrides for Racket extensions""" + """Provides appropriate overrides for Racket extensions""" base_class_name = "RacketPackage" package_class_import = "from spack_repo.builtin.build_systems.racket import RacketPackage" @@ -338,8 +340,9 @@ class RacketPackageTemplate(PackageTemplate): dependencies = """\ # FIXME: Add dependencies if required. Only add the racket dependency # if you need specific versions. A generic racket dependency is - # added implicity by the RacketPackage class. - # depends_on("racket@8.3:", type=("build", "run"))""" + # added implicitly by the RacketPackage class. + # with default_args(type=("build", "run")): + # depends_on("racket@8.3:")""" body_def = """\ # FIXME: specify the name of the package, @@ -371,20 +374,22 @@ class PythonPackageTemplate(PackageTemplate): dependencies = """\ # FIXME: Only add the python/pip/wheel dependencies if you need specific versions # or need to change the dependency type. Generic python/pip/wheel dependencies are - # added implicity by the PythonPackage base class. + # added implicitly by the PythonPackage base class. # depends_on("python@2.X:2.Y,3.Z:", type=("build", "run")) # depends_on("py-pip@X.Y:", type="build") # depends_on("py-wheel@X.Y:", type="build") # FIXME: Add a build backend, usually defined in pyproject.toml. If no such file # exists, use setuptools. - # depends_on("py-setuptools", type="build") - # depends_on("py-hatchling", type="build") - # depends_on("py-flit-core", type="build") - # depends_on("py-poetry-core", type="build") + # with default_args(type="build"): + # depends_on("py-setuptools") + # depends_on("py-hatchling") + # depends_on("py-flit-core") + # depends_on("py-poetry-core") # FIXME: Add additional dependencies if required. - # depends_on("py-foo", type=("build", "run"))""" + # with default_args(type=("build", "run")): + # depends_on("py-foo")""" body_def = """\ def config_settings(self, spec, prefix): @@ -409,7 +414,7 @@ def __init__(self, name, url, versions, languages: List[str]): # e.g. https://files.pythonhosted.org/packages/source/n/numpy/numpy-1.19.4.zip # PyPI URLs containing hash: - # https:///packages//// + # https:///packages//// # noqa: E501 # e.g. https://pypi.io/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip # e.g. https://files.pythonhosted.org/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip # e.g. https://files.pythonhosted.org/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip#sha256=141ec3a3300ab89c7f2b0775289954d193cc8edb621ea05f99db9cb181530512 @@ -458,7 +463,8 @@ class RPackageTemplate(PackageTemplate): dependencies = """\ # FIXME: Add dependencies if required. - # depends_on("r-foo", type=("build", "run"))""" + # with default_args(type=("build", "run")): + # depends_on("r-foo")""" body_def = """\ def configure_args(self): @@ -485,7 +491,7 @@ def __init__(self, name, url, versions, languages: List[str]): bioc = re.search(r"(?:bioconductor)[^/]+/packages" + "/([^/]+)" * 5, url) if bioc: - self.url_line = ' url = "{0}"\n' ' bioc = "{1}"'.format(url, r_name) + self.url_line = ' url = "{0}"\n bioc = "{1}"'.format(url, r_name) super().__init__(name, url, versions, languages) @@ -499,7 +505,8 @@ class PerlmakePackageTemplate(PackageTemplate): dependencies = """\ # FIXME: Add dependencies if required: - # depends_on("perl-foo", type=("build", "run"))""" + # with default_args(type=("build", "run")): + # depends_on("perl-foo")""" body_def = """\ def configure_args(self): @@ -526,7 +533,8 @@ class PerlbuildPackageTemplate(PerlmakePackageTemplate): depends_on("perl-module-build", type="build") # FIXME: Add additional dependencies if required: - # depends_on("perl-foo", type=("build", "run"))""" + # with default_args(type=("build", "run")): + # depends_on("perl-foo")""" class OctavePackageTemplate(PackageTemplate): @@ -539,7 +547,8 @@ class OctavePackageTemplate(PackageTemplate): extends("octave") # FIXME: Add additional dependencies if required. - # depends_on("octave-foo", type=("build", "run"))""" + # with default_args(type=("build", "run")): + # depends_on("octave-foo")""" def __init__(self, name, url, versions, languages: List[str]): # If the user provided `--name octave-splines`, don't rename it @@ -561,9 +570,10 @@ class RubyPackageTemplate(PackageTemplate): dependencies = """\ # FIXME: Add dependencies if required. Only add the ruby dependency # if you need specific versions. A generic ruby dependency is - # added implicity by the RubyPackage class. - # depends_on("ruby@X.Y.Z:", type=("build", "run")) - # depends_on("ruby-foo", type=("build", "run"))""" + # added implicitly by the RubyPackage class. + # with default_args(type=("build", "run")): + # depends_on("ruby@X.Y.Z:") + # depends_on("ruby-foo")""" body_def = """\ def build(self, spec, prefix): @@ -1051,8 +1061,9 @@ def get_repository(args: argparse.Namespace, name: str) -> spack.repo.Repo: repo = spack.repo.from_path(repo_path) if spec.namespace and spec.namespace != repo.namespace: tty.die( - "Can't create package with namespace {0} in repo with " - "namespace {1}".format(spec.namespace, repo.namespace) + "Can't create package with namespace {0} in repo with namespace {1}".format( + spec.namespace, repo.namespace + ) ) else: if spec.namespace: diff --git a/lib/spack/spack/cmd/deconcretize.py b/lib/spack/spack/cmd/deconcretize.py index d1afec66bd86df..78c087cb532d50 100644 --- a/lib/spack/spack/cmd/deconcretize.py +++ b/lib/spack/spack/cmd/deconcretize.py @@ -74,7 +74,7 @@ def get_deconcretize_list( def deconcretize_specs(args, specs): - env = spack.cmd.require_active_env(cmd_name="deconcretize") + env = spack.cmd.require_active_env(args.subparser) if args.specs: deconcretize_list = get_deconcretize_list(args, specs, env) @@ -86,15 +86,15 @@ def deconcretize_specs(args, specs): with env.write_transaction(): for spec in deconcretize_list: - env.deconcretize(spec) + env.deconcretize_by_hash(spec.dag_hash()) env.write() def deconcretize(parser, args): if not args.specs and not args.all: - tty.die( - "deconcretize requires at least one spec argument.", - " Use `spack deconcretize --all` to deconcretize ALL specs.", + args.subparser.error( + "requires at least one spec argument\n" + " use `spack deconcretize --all` to deconcretize ALL specs" ) specs = spack.cmd.parse_specs(args.specs) if args.specs else [None] diff --git a/lib/spack/spack/cmd/dependencies.py b/lib/spack/spack/cmd/dependencies.py index 5a6055f396131f..dc066b2ecfbbca 100644 --- a/lib/spack/spack/cmd/dependencies.py +++ b/lib/spack/spack/cmd/dependencies.py @@ -49,7 +49,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: def dependencies(parser, args): specs = spack.cmd.parse_specs(args.spec) if len(specs) != 1: - tty.die("spack dependencies takes only one spec.") + args.subparser.error("takes only one spec") if args.installed: env = ev.active_environment() diff --git a/lib/spack/spack/cmd/dependents.py b/lib/spack/spack/cmd/dependents.py index 4591489404e808..9b8fc70a42381d 100644 --- a/lib/spack/spack/cmd/dependents.py +++ b/lib/spack/spack/cmd/dependents.py @@ -87,7 +87,7 @@ def get_dependents(pkg_name, ideps, transitive=False, dependents=None): def dependents(parser, args): specs = spack.cmd.parse_specs(args.spec) if len(specs) != 1: - tty.die("spack dependents takes only one spec.") + args.subparser.error("takes only one spec") if args.installed: env = ev.active_environment() diff --git a/lib/spack/spack/cmd/deprecate.py b/lib/spack/spack/cmd/deprecate.py index b92b813fc90dc4..5bb55bb994b3f6 100644 --- a/lib/spack/spack/cmd/deprecate.py +++ b/lib/spack/spack/cmd/deprecate.py @@ -12,6 +12,7 @@ It is up to the user to ensure binary compatibility between the deprecated installation and its deprecator. """ + import argparse import spack.cmd @@ -21,7 +22,6 @@ import spack.llnl.util.tty as tty import spack.store from spack.cmd.common import arguments -from spack.error import SpackError from spack.llnl.util.filesystem import symlink from ..enums import InstallRecordStatus @@ -93,7 +93,7 @@ def deprecate(parser, args): specs = spack.cmd.parse_specs(args.specs) if len(specs) != 2: - raise SpackError("spack deprecate requires exactly two specs") + args.subparser.error("requires exactly two specs") deprecate = spack.cmd.disambiguate_spec( specs[0], diff --git a/lib/spack/spack/cmd/dev_build.py b/lib/spack/spack/cmd/dev_build.py index 26b03324bb99d0..3bd89ea0098862 100644 --- a/lib/spack/spack/cmd/dev_build.py +++ b/lib/spack/spack/cmd/dev_build.py @@ -11,10 +11,10 @@ import spack.cmd.common.arguments import spack.concretize import spack.config +import spack.installer_dispatch import spack.llnl.util.tty as tty import spack.repo from spack.cmd.common import arguments -from spack.installer import PackageInstaller description = "build package from code in current working directory" section = "build" @@ -92,20 +92,19 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: def dev_build(self, args): if not args.spec: - tty.die("spack dev-build requires a package spec argument.") + args.subparser.error("requires a package spec argument") specs = spack.cmd.parse_specs(args.spec) if len(specs) > 1: - tty.die("spack dev-build only takes one spec.") + args.subparser.error("only takes one spec") spec = specs[0] if not spack.repo.PATH.exists(spec.name): raise spack.repo.UnknownPackageError(spec.name) if not spec.versions.concrete_range_as_version: - tty.die( - "spack dev-build spec must have a single, concrete version. " - "Did you forget a package version number?" + args.subparser.error( + "spec must have a single, concrete version. Did you forget a package version number?" ) source_path = args.source_path @@ -132,7 +131,7 @@ def dev_build(self, args): elif args.test == "root": tests = [spec.name for spec in specs] - PackageInstaller( + spack.installer_dispatch.create_installer( [spec.package], tests=tests, keep_prefix=args.keep_prefix, diff --git a/lib/spack/spack/cmd/develop.py b/lib/spack/spack/cmd/develop.py index ec29462dbdfa33..025c690fa91fe3 100644 --- a/lib/spack/spack/cmd/develop.py +++ b/lib/spack/spack/cmd/develop.py @@ -108,7 +108,7 @@ def assure_concrete_spec(env: spack.environment.Environment, spec: spack.spec.Sp if not m_spec.satisfies(test_spec): raise SpackError( f"{spec.name}: has multiple concrete instances in the graph that can't be" - " satisified by a single develop spec. To use `spack develop` ensure one" + " satisfied by a single develop spec. To use `spack develop` ensure one" " of the following:" f"\n a) {spec.name} nodes can satisfy the same develop spec (minimally " "this means they all share the same version)" @@ -183,7 +183,7 @@ def update_env( # If we are automatically mutating the concrete specs for dev provenance, do so if apply_changes: - env.apply_develop(spec, _abs_code_path(env, spec, specified_path)) + env.apply_develop([spec], [_abs_code_path(env, spec, specified_path)]) def _clone(spec: spack.spec.Spec, abspath: str, force: bool = False): @@ -215,7 +215,7 @@ def _dev_spec_generator(args, env): """ if not args.spec: if args.clone is False: - raise SpackError("No spec provided to spack develop command") + args.subparser.error("no spec provided") for name, entry in env.dev_specs.items(): path = entry.get("path", name) @@ -227,9 +227,9 @@ def _dev_spec_generator(args, env): else: specs = spack.cmd.parse_specs(args.spec) if (args.path or args.build_directory) and len(specs) > 1: - raise SpackError( - "spack develop requires at most one named spec when using the --path or" - " --build-directory arguments" + args.subparser.error( + "requires at most one named spec when using the --path or --build-directory " + "arguments" ) for spec in specs: @@ -252,7 +252,7 @@ def _dev_spec_generator(args, env): def develop(parser, args): - env = spack.cmd.require_active_env(cmd_name="develop") + env = spack.cmd.require_active_env(args.subparser) for spec, abspath in _dev_spec_generator(args, env): assure_concrete_spec(env, spec) diff --git a/lib/spack/spack/cmd/diff.py b/lib/spack/spack/cmd/diff.py index a0269f589b7308..fa889284205f44 100644 --- a/lib/spack/spack/cmd/diff.py +++ b/lib/spack/spack/cmd/diff.py @@ -209,7 +209,7 @@ def diff(parser, args): env = ev.active_environment() if len(args.specs) != 2: - tty.die("You must provide two specs to diff.") + args.subparser.error("you must provide two specs to diff") specs = [] for spec in spack.cmd.parse_specs(args.specs): @@ -228,7 +228,7 @@ def diff(parser, args): attributes = args.attribute or ["all"] if args.dump_json: - print(sjson.dump(c)) + print(sjson.dumps(c)) else: tty.warn("This interface is subject to change.\n") print_difference(c, attributes) diff --git a/lib/spack/spack/cmd/env.py b/lib/spack/spack/cmd/env.py index 5f75d1b328084d..e324a0a0088c00 100644 --- a/lib/spack/spack/cmd/env.py +++ b/lib/spack/spack/cmd/env.py @@ -28,6 +28,7 @@ from spack.llnl.util.filesystem import islink, symlink from spack.llnl.util.tty.colify import colify from spack.llnl.util.tty.color import cescape, colorize +from spack.traverse import traverse_nodes from spack.util.environment import EnvironmentModifications description = "manage environments" @@ -126,7 +127,8 @@ def env_create(args): ) # Generate views, only really useful for environments created from spack.lock files. - env.regenerate_views() + if args.envfile: + env.regenerate_views() def _env_create( @@ -141,15 +143,15 @@ def _env_create( """Create a new environment, with an optional yaml description. Arguments: - name_or_path (str): name of the environment to create, or path to it - init_file (str or file): optional initialization file -- can be - a JSON lockfile (*.lock, *.json), YAML manifest file, or env dir - dir (bool): if True, create an environment in a directory instead - of a named environment - keep_relative (bool): if True, develop paths are copied verbatim into - the new environment file, otherwise they may be made absolute if the - new environment is in a different location - include_concrete (list): list of the included concrete environments + name_or_path: name of the environment to create, or path to it + init_file: optional initialization file -- can be a JSON lockfile + (*.lock, *.json), YAML manifest file, or env dir + dir: if True, create an environment in a directory instead of a named + environment + keep_relative: if True, develop paths are copied verbatim into the new + environment file, otherwise they may be made absolute if the new + environment is in a different location + include_concrete: list of the included concrete environments """ if not dir: env = ev.create( @@ -546,7 +548,7 @@ def _env_untrack_or_remove( else: env_names_to_remove = known_env_names - # initalize all environments with valid spack.yaml configs + # initialize all environments with valid spack.yaml configs all_valid_envs = get_valid_envs(all_env_names) # build a task list of environments and bad env names to remove @@ -558,8 +560,8 @@ def _env_untrack_or_remove( if env.name == remove_env.name: continue - # check if an environment is included un another - if remove_env.path in env.included_concrete_envs: + # check if an environment is included in another + if remove_env.path in env.included_concrete_env_root_dirs: msg = f"Environment '{remove_env.name}' is used by environment '{env.name}'" if force: tty.warn(msg) @@ -568,7 +570,7 @@ def _env_untrack_or_remove( envs_to_remove.remove(remove_env) # ask the user if they really want to remove the known environments - # force should do the same as yes to all here following the symantics of rm + # force should do the same as yes to all here following the semantics of rm if not (yes_to_all or force) and (envs_to_remove or bad_env_names_to_remove): environments = string.plural(len(env_names_to_remove), "environment", show_n=False) envs = string.comma_and(list(env_names_to_remove)) @@ -594,7 +596,7 @@ def _env_untrack_or_remove( real_env_path = os.path.realpath(env.path) os.unlink(env.path) tty.msg( - f"Sucessfully untracked environment '{name}', " + f"Successfully untracked environment '{name}', " "but it can still be found at:\n\n" f" {real_env_path}\n" ) @@ -614,7 +616,7 @@ def _env_untrack_or_remove( # Following the design of linux rm we should exit with a status of 1 # anytime we cannot delete every environment the user asks for. # However, we should still process all the environments we know about - # and delete them instead of failing on the first unknown enviornment. + # and delete them instead of failing on the first unknown environment. if len(removed_env_names) < len(known_env_names): sys.exit(1) @@ -857,7 +859,7 @@ def env_loads_setup_parser(subparser): def env_loads(args): - env = spack.cmd.require_active_env(cmd_name="env loads") + env = spack.cmd.require_active_env(args.subparser) # Set the module types that have been selected module_type = args.module_type @@ -870,8 +872,10 @@ def env_loads(args): loads_file = fs.join_path(env.path, "loads") with open(loads_file, "w", encoding="utf-8") as f: - specs = env._get_environment_specs(recurse_dependencies=recurse_dependencies) - + if not recurse_dependencies: + specs = [env.specs_by_hash[x.hash] for x in env.concretized_roots] + else: + specs = list(traverse_nodes(env.concrete_roots(), deptype=("link", "run"))) spack.cmd.modules.loads(module_type, specs, args, f) print("To load this environment, type:") @@ -1029,7 +1033,7 @@ def env_depfile_setup_parser(subparser): def env_depfile(args): # Currently only make is supported. - spack.cmd.require_active_env(cmd_name="env depfile") + spack.cmd.require_active_env(args.subparser) env = ev.active_environment() @@ -1094,6 +1098,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: description=spack.cmd.doc_dedented(setup_parser_cmd), help=spack.cmd.doc_first_line(setup_parser_cmd), ) + subsubparser.set_defaults(subparser=subsubparser) setup_parser_cmd(subsubparser) diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index b8d80fa0f9d95b..9b0df83dec7840 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -66,7 +66,7 @@ def extensions(parser, args): # Checks spec = cmd.parse_specs(args.spec) if len(spec) > 1: - tty.die("Can only list extensions for one package.") + args.subparser.error("can only list extensions for one package") env = ev.active_environment() spec = cmd.disambiguate_spec(spec[0], env) diff --git a/lib/spack/spack/cmd/fetch.py b/lib/spack/spack/cmd/fetch.py index 66d3c7a01b8934..634b90f0f2d1bd 100644 --- a/lib/spack/spack/cmd/fetch.py +++ b/lib/spack/spack/cmd/fetch.py @@ -7,7 +7,6 @@ import spack.cmd import spack.config import spack.environment as ev -import spack.llnl.util.tty as tty import spack.traverse from spack.cmd.common import arguments @@ -54,9 +53,11 @@ def fetch(parser, args): else: specs = env.all_specs() if specs == []: - tty.die("No uninstalled specs in environment. Did you run `spack concretize` yet?") + args.subparser.error( + "no uninstalled specs in environment. Did you run `spack concretize` yet?" + ) else: - tty.die("fetch requires at least one spec argument") + args.subparser.error("requires at least one spec argument") if args.dependencies or args.missing: to_be_fetched = spack.traverse.traverse_nodes(specs, key=spack.traverse.by_dag_hash) diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 9d877eb51dbde3..682f0a21f7b509 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -17,6 +17,7 @@ import spack.spec import spack.store from spack.cmd.common import arguments +from spack.llnl.util.tty.color import colorize from spack.solver.reuse import create_external_parser from spack.solver.runtimes import external_config_with_implicit_externals @@ -188,10 +189,10 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: def query_arguments(args): if args.only_missing and (args.deprecated or args.missing): - raise RuntimeError("cannot use --only-missing with --deprecated, or --missing") + args.subparser.error("cannot use --only-missing with --deprecated, or --missing") if args.only_deprecated and (args.deprecated or args.missing): - raise RuntimeError("cannot use --only-deprecated with --deprecated, or --missing") + args.subparser.error("cannot use --only-deprecated with --deprecated, or --missing") installed = InstallRecordStatus.INSTALLED if args.only_missing: @@ -256,17 +257,12 @@ def display_env(env, args, decorator, results): In an environment, ``spack find`` outputs a preliminary section showing the root specs of the environment (this is in addition to the section listing out specs matching the query parameters). - """ - tty.msg("In environment %s" % env.name) - - num_roots = len(env.user_specs) or "No" - tty.msg(f"{num_roots} root specs") + total_roots = sum(len(env.user_specs_by(group=g)) for g in env.manifest.groups()) + root_spec_str = f"{total_roots or 'no'} root {'spec' if total_roots == 1 else 'specs'}" + tty.msg(f"In environment {env.name} ({root_spec_str})") - concrete_specs = { - root: concrete_root - for root, concrete_root in zip(env.concretized_user_specs, env.concrete_roots()) - } + concrete_specs = {x.root: env.specs_by_hash[x.hash] for x in env.concretized_roots} def root_decorator(spec, string): """Decorate root specs with their install status if needed""" @@ -289,24 +285,36 @@ def root_decorator(spec, string): return f"{status} {string}" with spack.store.STORE.db.read_transaction(): - cmd.display_specs( - env.user_specs, - args, - # these are overrides of CLI args - paths=False, - long=False, - very_long=False, - # these enforce details in the root specs to show what the user asked for - namespaces=True, - show_flags=True, - decorator=root_decorator, - variants=True, - specfile_format=args.specfile_format, - ) + for group in env.manifest.groups(): + group_specs = env.user_specs_by(group=group) + if not group_specs: + continue + + if env.has_groups(): + header = ( + f"{spack.spec.ARCHITECTURE_COLOR}{{root specs}} / " + f"{spack.spec.COMPILER_COLOR}{{{group}}}" + ) + tty.hline(colorize(header), char="-") - print() + cmd.display_specs( + group_specs, + args, + # these are overrides of CLI args + paths=False, + long=False, + very_long=False, + # these enforce details in the root specs to show what the user asked for + groups=False, + namespaces=True, + show_flags=True, + decorator=root_decorator, + variants=True, + specfile_format=args.specfile_format, + ) + print() - if env.included_concrete_envs: + if env.included_concrete_env_root_dirs: tty.msg("Included specs") # Root specs cannot be displayed with prefixes, since those are not @@ -334,7 +342,7 @@ def _find_query(args, env): if args.show_configured_externals: packages_with_externals = external_config_with_implicit_externals(spack.config.CONFIG) completion_mode = spack.config.CONFIG.get("concretizer:externals:completion") - results = spack.solver.reuse.SpecFilter.from_packages_yaml( + results = spack.solver.reuse.spec_filter_from_packages_yaml( external_parser=create_external_parser(packages_with_externals, completion_mode), packages_with_externals=packages_with_externals, include=[], @@ -394,9 +402,9 @@ def find(parser, args): env = ev.active_environment() if not env and args.only_roots: - tty.die("-r / --only-roots requires an active environment") + args.subparser.error("-r / --only-roots requires an active environment") if not env and args.show_concretized: - tty.die("-c / --show-concretized requires an active environment") + args.subparser.error("-c / --show-concretized requires an active environment") try: results, concretized_but_not_installed = _find_query(args, env) diff --git a/lib/spack/spack/cmd/gc.py b/lib/spack/spack/cmd/gc.py index 72d9a126498915..e303df5cf2f094 100644 --- a/lib/spack/spack/cmd/gc.py +++ b/lib/spack/spack/cmd/gc.py @@ -67,7 +67,7 @@ def roots_from_environments(args, active_env): # add root hashes from all considered environments to list of roots root_hashes = set() for env in all_environments: - root_hashes |= set(env.concretized_order) + root_hashes |= {x.hash for x in env.explicit_roots()} return root_hashes @@ -91,7 +91,8 @@ def gc(parser, args): tty.msg(f"Restricting garbage collection to environment '{active_env.name}'") root_hashes = set(spack.store.STORE.db.all_hashes()) # keep everything root_hashes -= set(active_env.all_hashes()) # except this env - root_hashes |= set(active_env.concretized_order) # but keep its roots + # but keep its explicit roots + root_hashes |= {x.hash for x in active_env.explicit_roots()} else: # consider all explicit specs roots (the default for db.unused_specs()) root_hashes = None diff --git a/lib/spack/spack/cmd/gpg.py b/lib/spack/spack/cmd/gpg.py index 66c9d9a41a64f4..3f1f5233f6cde5 100644 --- a/lib/spack/spack/cmd/gpg.py +++ b/lib/spack/spack/cmd/gpg.py @@ -26,16 +26,16 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: verify = subparsers.add_parser("verify", help=gpg_verify.__doc__) arguments.add_common_arguments(verify, ["installed_spec"]) verify.add_argument("signature", type=str, nargs="?", help="the signature file") - verify.set_defaults(func=gpg_verify) + verify.set_defaults(func=gpg_verify, subparser=verify) trust = subparsers.add_parser("trust", help=gpg_trust.__doc__) trust.add_argument("keyfile", type=str, help="add a key to the trust store") - trust.set_defaults(func=gpg_trust) + trust.set_defaults(func=gpg_trust, subparser=trust) untrust = subparsers.add_parser("untrust", help=gpg_untrust.__doc__) untrust.add_argument("--signing", action="store_true", help="allow untrusting signing keys") untrust.add_argument("keys", nargs="+", type=str, help="remove keys from the trust store") - untrust.set_defaults(func=gpg_untrust) + untrust.set_defaults(func=gpg_untrust, subparser=untrust) sign = subparsers.add_parser("sign", help=gpg_sign.__doc__) sign.add_argument( @@ -46,7 +46,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: "--clearsign", action="store_true", help="if specified, create a clearsign signature" ) arguments.add_common_arguments(sign, ["installed_spec"]) - sign.set_defaults(func=gpg_sign) + sign.set_defaults(func=gpg_sign, subparser=sign) create = subparsers.add_parser("create", help=gpg_create.__doc__) create.add_argument("name", type=str, help="the name to use for the new key") @@ -71,18 +71,18 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: dest="secret", help="export the private key to a file", ) - create.set_defaults(func=gpg_create) + create.set_defaults(func=gpg_create, subparser=create) list = subparsers.add_parser("list", help=gpg_list.__doc__) list.add_argument("--trusted", action="store_true", default=True, help="list trusted keys") list.add_argument( "--signing", action="store_true", help="list keys which may be used for signing" ) - list.set_defaults(func=gpg_list) + list.set_defaults(func=gpg_list, subparser=list) init = subparsers.add_parser("init", help=gpg_init.__doc__) init.add_argument("--from", metavar="DIR", type=str, dest="import_dir", help=argparse.SUPPRESS) - init.set_defaults(func=gpg_init) + init.set_defaults(func=gpg_init, subparser=init) export = subparsers.add_parser("export", help=gpg_export.__doc__) export.add_argument("location", type=str, help="where to export keys") @@ -90,7 +90,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: "keys", nargs="*", help="the keys to export (all public keys if unspecified)" ) export.add_argument("--secret", action="store_true", help="export secret keys") - export.set_defaults(func=gpg_export) + export.set_defaults(func=gpg_export, subparser=export) publish = subparsers.add_parser("publish", help=gpg_publish.__doc__) @@ -125,7 +125,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: publish.add_argument( "keys", nargs="*", help="keys to publish (all public keys if unspecified)" ) - publish.set_defaults(func=gpg_publish) + publish.set_defaults(func=gpg_publish, subparser=publish) def gpg_create(args): diff --git a/lib/spack/spack/cmd/graph.py b/lib/spack/spack/cmd/graph.py index de310e72b0b3fb..f63b2b561f8cc4 100644 --- a/lib/spack/spack/cmd/graph.py +++ b/lib/spack/spack/cmd/graph.py @@ -57,10 +57,10 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: def graph(parser, args): env = ev.active_environment() if args.installed and env: - tty.die("cannot use --installed with an active environment") + args.subparser.error("cannot use --installed with an active environment") if args.color and not args.dot: - tty.die("the --color option can be used only with --dot") + args.subparser.error("the --color option can be used only with --dot") if args.installed: if not args.specs: diff --git a/lib/spack/spack/cmd/info.py b/lib/spack/spack/cmd/info.py index 9ea09d38114c3f..51a42f14b2f558 100644 --- a/lib/spack/spack/cmd/info.py +++ b/lib/spack/spack/cmd/info.py @@ -639,9 +639,9 @@ def print_virtuals(pkg: PackageBase, args: Namespace) -> None: def info(parser: argparse.ArgumentParser, args: Namespace) -> None: specs = spack.cmd.parse_specs(args.spec) if len(specs) > 1: - tty.die(f"`spack info` requires exactly one spec. Parsed {len(specs)}") + args.subparser.error(f"requires exactly one spec, got {len(specs)}") if len(specs) == 0: - tty.die("`spack info` requires a spec.") + args.subparser.error("requires a spec") spec = specs[0] pkg_cls = spack.repo.PATH.get_pkg_class(spec.fullname) diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 10a097f65d63fd..d610e090ebe086 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -11,6 +11,7 @@ import spack.cmd import spack.config import spack.environment as ev +import spack.installer_dispatch import spack.llnl.util.filesystem as fs import spack.paths import spack.spec @@ -286,8 +287,8 @@ def _dump_log_on_error(e: InstallError): shutil.copyfileobj(log, sys.stderr) -def _die_require_env(): - msg = "install requires a package argument or active environment" +def _die_require_env(parser): + msg = "requires a package argument or active environment" if "spack.yaml" in os.listdir(os.getcwd()): # There's a spack.yaml file in the working dir, the user may # have intended to use that @@ -300,7 +301,7 @@ def _die_require_env(): " OR\n" " spack --env . install" ) - tty.die(msg) + parser.error(msg) def install(parser, args): @@ -325,7 +326,7 @@ def install(parser, args): env = ev.active_environment() if not env and not args.spec: - _die_require_env() + _die_require_env(args.subparser) try: if env: @@ -430,7 +431,7 @@ def install_without_active_env(args, install_kwargs, reporter): concrete_specs = concrete_specs_from_cli(args, install_kwargs) if len(concrete_specs) == 0: - tty.die("The `spack install` command requires a spec to install.") + args.subparser.error("requires a spec") if args.overwrite: require_user_confirmation_for_overwrite(concrete_specs, args) @@ -439,13 +440,10 @@ def install_without_active_env(args, install_kwargs, reporter): installs = [s.package for s in concrete_specs] install_kwargs["explicit"] = [s.dag_hash() for s in concrete_specs] - if spack.config.get("config:installer", "old") == "new": - from spack.new_installer import PackageInstaller - else: - from spack.installer import PackageInstaller - try: - builder = PackageInstaller(installs, **install_kwargs) + builder = spack.installer_dispatch.create_installer( + installs, create_reports=reporter is not None, **install_kwargs + ) builder.install() finally: if reporter: diff --git a/lib/spack/spack/cmd/installer/CMakeLists.txt b/lib/spack/spack/cmd/installer/CMakeLists.txt index efa9f2b6df9711..0c9e5638fb6556 100644 --- a/lib/spack/spack/cmd/installer/CMakeLists.txt +++ b/lib/spack/spack/cmd/installer/CMakeLists.txt @@ -11,7 +11,7 @@ if (SPACK_VERSION) set(SPACK_FILENAME "spack-${SPACK_VERSION}.tar.gz") set(SPACK_DIR "spack-${SPACK_VERSION}") - # SPACK DOWLOAD AND EXTRACTION----------------------------------- + # SPACK DOWNLOAD AND EXTRACTION----------------------------------- file(DOWNLOAD "${SPACK_DL}/${SPACK_FILENAME}" "${CMAKE_CURRENT_BINARY_DIR}/${SPACK_FILENAME}" STATUS download_status @@ -54,7 +54,7 @@ endif() message(STATUS "Successfully downloaded ${GIT_FILENAME}") -# PYTHON DOWLOAD AND EXTRACTION----------------------------------- +# PYTHON DOWNLOAD AND EXTRACTION----------------------------------- file(DOWNLOAD "${PY_DOWNLOAD_LINK}/${PY_FILENAME}" "${CMAKE_CURRENT_BINARY_DIR}/${PY_FILENAME}" STATUS download_status diff --git a/lib/spack/spack/cmd/installer/README.md b/lib/spack/spack/cmd/installer/README.md index 602c594f093671..53566871d7168a 100644 --- a/lib/spack/spack/cmd/installer/README.md +++ b/lib/spack/spack/cmd/installer/README.md @@ -73,7 +73,7 @@ install. When given the option of adjusting your ``PATH``, choose the ``Git from the command line and also from 3rd-party software`` option. This will automatically update your ``PATH`` variable to include the ``git`` command. Certain Spack commands expect ``git`` to be part of the ``PATH``. If this step -is not performed properly, certain Spack comands will not work. +is not performed properly, certain Spack commands will not work. If your Spack installation needs to be modified, repaired, or uninstalled, you can do any of these things by rerunning ``Spack.exe``. diff --git a/lib/spack/spack/cmd/license.py b/lib/spack/spack/cmd/license.py index 09fa5ba7a31722..dd8a970383fb60 100644 --- a/lib/spack/spack/cmd/license.py +++ b/lib/spack/spack/cmd/license.py @@ -33,7 +33,6 @@ r"^bin/spack_pwsh\.ps1$", r"^bin/sbang$", r"^bin/spack-python$", - r"^bin/haspywin\.py$", # all of spack core except unparse r"^lib/spack/spack/(?!vendor/|util/unparse|util/ctest_log_parser|test/util/unparse).*\.py$", r"^lib/spack/spack/.*\.sh$", diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index eaa63f3ebeb02f..1a341ef7b0060c 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -10,14 +10,17 @@ import re import sys from html import escape -from typing import Type +from typing import Optional, Type import spack.deptypes as dt import spack.llnl.util.tty as tty import spack.package_base import spack.repo +import spack.util.git from spack.cmd.common import arguments +from spack.llnl.util.filesystem import working_dir from spack.llnl.util.tty.colify import colify +from spack.util.url import path_to_file_url from spack.version import VersionList description = "list and search available packages" @@ -140,10 +143,60 @@ def name_only(pkgs, out): tty.msg("%d packages" % len(pkgs)) -def github_url(pkg: Type[spack.package_base.PackageBase]) -> str: - """Link to a package file on github.""" - mod_path = pkg.__module__.replace(".", "/") - return f"https://github.com/spack/spack/blob/develop/var/spack/{mod_path}.py" +def github_url(pkg: Type[spack.package_base.PackageBase]) -> Optional[str]: + """Link to a package file in spack package's github or the path to the file. + + Args: + pkg: package instance + + Returns: URL to the package file on github or the local file path; otherwise, ``None``. + """ + git = None + module_path = f"{pkg.__module__.replace('.', '/')}.py" + for repo in spack.repo.PATH.repos: + if not repo.python_path: + continue + + path = os.path.join(repo.python_path, module_path) + if not os.path.exists(path): + continue + + git = git or spack.util.git.git() + if not git: + tty.debug("Cannot determine package URL for {pkg} without 'git', using path URL") + return path_to_file_url(path) + + tty.debug(f"Checking git for repository path '{path}'") + with working_dir(os.path.dirname(path)): + origin_url = git( + "config", + "--get", + "remote.origin.url", + output=str, + error=os.devnull, + fail_on_error=False, + ) + + if not origin_url: + tty.debug("Cannot determine remote origin url, using path URL") + return path_to_file_url(path) + + # Handle spack repositories cloned with any scheme (e.g., ssh) by + # ignoring the scheme designation. + if any([name in origin_url for name in ["spack.git", "spack-packages.git"]]): + git_repo = (origin_url.split("/")[-1]).replace(".git", "").strip() + prefix = git( + "rev-parse", "--show-prefix", output=str, error=os.devnull, fail_on_error=False + ) + return ( + f"https://github.com/spack/{git_repo}/blob/develop/{prefix.strip()}package.py" + ) + + tty.debug(f"Unrecognized repository for {pkg}, using path URL") + return path_to_file_url(path) + + tty.debug(f"Unable to determine the package repository URL for {pkg}") + return None def rows_for_ncols(elts, ncols): @@ -263,7 +316,7 @@ def head(n, span_id, title, anchor=None): if pkg_cls.homepage: out.write( - ("
  • " '%s' "
    • \n") + ('
  • %s
    • \n') % (pkg_cls.homepage, escape(pkg_cls.homepage, True)) ) else: @@ -273,7 +326,7 @@ def head(n, span_id, title, anchor=None): out.write("
    Spack package:
    \n") out.write('
    \n") diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index 9d23aa6d9427e9..96571ff20341c9 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -79,6 +79,16 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: default=False, help="location of the named or current environment", ) + directories.add_argument( + "-v", + "--view", + action="store", + nargs="?", + metavar="name", + dest="location_view", + default=False, + help="location of the named or active environment view", + ) subparser.add_argument( "--first", @@ -104,7 +114,7 @@ def location(parser, args): if args.location_env is not False: if args.location_env is None: # Get current environment path - spack.cmd.require_active_env("location -e") + spack.cmd.require_active_env(args.subparser) path = ev.active_environment().path else: # Get path of requested environment @@ -114,6 +124,22 @@ def location(parser, args): print(path) return + # no -v corresponds to False, -v without arg to None, -v name to the string name. + if args.location_view is not False: + env = spack.cmd.require_active_env("location -v") + view_name = args.location_view + if view_name is None: + # get active view name + view_name = os.getenv(ev.spack_env_view_var) + if view_name is None: + tty.die("no active view in the current environment") + # print the view location + if env.has_view(view_name): + print(f"{env.views[view_name].root}\n") + else: + tty.die("no such view in the current environment: '%s'" % view_name) + return + if args.repo is not False: if args.repo is None: print(spack.repo.PATH.first_repo().root) @@ -131,10 +157,10 @@ def location(parser, args): specs = spack.cmd.parse_specs(args.spec) if not specs: - tty.die("You must supply a spec.") + args.subparser.error("requires a spec") if len(specs) != 1: - tty.die("Too many specs. Supply only one.") + args.subparser.error("too many specs, supply only one") # install_dir command matches against installed specs. if args.install_dir: diff --git a/lib/spack/spack/cmd/log_parse.py b/lib/spack/spack/cmd/log_parse.py index 910908c246329d..c4ddf0321563f9 100644 --- a/lib/spack/spack/cmd/log_parse.py +++ b/lib/spack/spack/cmd/log_parse.py @@ -3,9 +3,10 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse +import io import sys +import warnings -import spack.llnl.util.tty as tty from spack.util.log_parse import make_log_context, parse_log_events description = "filter errors and warnings from build logs" @@ -37,21 +38,19 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: help="print out a profile of time spent in regexes during parse", ) subparser.add_argument( - "-w", - "--width", - action="store", - type=int, - default=None, - help="wrap width: auto-size to terminal by default; 0 for no wrap", + "-w", "--width", action="store", type=int, default=None, help=argparse.SUPPRESS + ) + subparser.add_argument( + "-j", "--jobs", action="store", type=int, default=None, help=argparse.SUPPRESS ) subparser.add_argument( - "-j", - "--jobs", + "-t", + "--tail", + metavar="LINES", action="store", type=int, - default=None, - help="number of jobs to parse log file (default: 1 for short logs, " - "ncpus for long logs)", + default=0, + help="number of trailing log lines to show (0 to disable)", ) subparser.add_argument("file", help="a log file containing build output, or - for stdin") @@ -60,23 +59,35 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: def log_parse(parser, args): input = args.file if args.file == "-": - input = sys.stdin + input = io.TextIOWrapper( + sys.stdin.buffer, encoding="utf-8", errors="replace", closefd=False + ) - errors, warnings = parse_log_events(input, args.context, args.jobs, args.profile) + if args.width is not None: + warnings.warn("The --width option is deprecated and will be removed in Spack v1.3") + if args.jobs is not None: + warnings.warn("The --jobs option is deprecated and will be removed in Spack v1.3") + + log_errors, log_warnings, tail = parse_log_events( + input, args.context, args.profile, tail=args.tail + ) if args.profile: return types = [s.strip() for s in args.show.split(",")] for e in types: if e not in event_types: - tty.die("Invalid event type: %s" % e) + args.subparser.error("invalid event type: %s" % e) events = [] if "errors" in types: - events.extend(errors) - print("%d errors" % len(errors)) + events.extend(log_errors) + print("%d errors" % len(log_errors)) if "warnings" in types: - events.extend(warnings) - print("%d warnings" % len(warnings)) + events.extend(log_warnings) + print("%d warnings" % len(log_warnings)) + + if tail: + events.append(tail) - print(make_log_context(events, args.width)) + print(make_log_context(events), end="") diff --git a/lib/spack/spack/cmd/logs.py b/lib/spack/spack/cmd/logs.py index 94db8798bd3dec..0a91c4bc5e9af6 100644 --- a/lib/spack/spack/cmd/logs.py +++ b/lib/spack/spack/cmd/logs.py @@ -61,10 +61,10 @@ def logs(parser, args): specs = spack.cmd.parse_specs(args.spec) if not specs: - raise SpackError("You must supply a spec.") + args.subparser.error("requires a spec") if len(specs) != 1: - raise SpackError("Too many specs. Supply only one.") + args.subparser.error("too many specs, supply only one") concrete_spec = spack.cmd.matching_spec_from_env(specs[0]) diff --git a/lib/spack/spack/cmd/maintainers.py b/lib/spack/spack/cmd/maintainers.py index 7d6afa4b891e54..4d2974c08e8335 100644 --- a/lib/spack/spack/cmd/maintainers.py +++ b/lib/spack/spack/cmd/maintainers.py @@ -5,7 +5,6 @@ import argparse from collections import defaultdict -import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as color import spack.repo from spack.llnl.util.tty.colify import colify @@ -123,7 +122,7 @@ def maintainers(parser, args): if args.by_user: if not args.package_or_user: - tty.die("spack maintainers --by-user requires a user or --all") + args.subparser.error("--by-user requires a user or --all") packages = union_values(maintainers_to_packages(args.package_or_user)) colify(packages) @@ -131,7 +130,7 @@ def maintainers(parser, args): else: if not args.package_or_user: - tty.die("spack maintainers requires a package or --all") + args.subparser.error("requires a package or --all") users = union_values(packages_to_maintainers(args.package_or_user)) colify(users) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 13c9ec8659697f..5f156e2fb1803e 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -364,7 +364,7 @@ def mirror_add(args): def mirror_remove(args): """remove a mirror by name""" name = args.name - scopes = [args.scope] if args.scope else list(spack.config.CONFIG.scopes.keys()) + scopes = [args.scope] if args.scope else reversed(list(spack.config.CONFIG.scopes.keys())) removed = False for scope in scopes: @@ -587,9 +587,10 @@ def versions_per_spec(args): try: num_versions = int(args.versions_per_spec) except ValueError: - raise SpackError( - "'--versions-per-spec' must be a number or 'all'," - " got '{0}'".format(args.versions_per_spec) + args.subparser.error( + "'--versions-per-spec' must be a number or 'all', got '{0}'".format( + args.versions_per_spec + ) ) return num_versions @@ -611,24 +612,24 @@ def process_mirror_stats(present, mirrored, error): def mirror_create(args): """create a directory to be used as a spack mirror, and fill it with package archives""" if args.file and args.all: - raise SpackError( + args.subparser.error( "cannot specify specs with a file if you chose to mirror all specs with '--all'" ) if args.file and args.specs: - raise SpackError("cannot specify specs with a file AND on command line") + args.subparser.error("cannot specify specs with a file AND on command line") if not args.specs and not args.file and not args.all: - raise SpackError( - "no packages were specified.", - "To mirror all packages, use the '--all' option " - "(this will require significant time and space).", + args.subparser.error( + "no packages were specified\n" + " to mirror all packages, use the '--all' option" + " (this will require significant time and space)" ) if args.versions_per_spec and args.all: - raise SpackError( - "cannot specify '--versions_per-spec' and '--all' together", - "The option '--all' already implies mirroring all versions for each package.", + args.subparser.error( + "cannot specify '--versions_per-spec' and '--all' together\n" + " '--all' already implies mirroring all versions for each package" ) # When no directory is provided, the source dir is used diff --git a/lib/spack/spack/cmd/patch.py b/lib/spack/spack/cmd/patch.py index 8f6522560f83bb..31ad6f9cf20deb 100644 --- a/lib/spack/spack/cmd/patch.py +++ b/lib/spack/spack/cmd/patch.py @@ -26,7 +26,7 @@ def patch(parser, args): if not args.specs: env = ev.active_environment() if not env: - tty.die("`spack patch` requires a spec or an active environment") + args.subparser.error("requires a spec or an active environment") return _patch_env(env) if args.no_checksum: diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py index 82a8eb40849553..dbb1bd14650482 100644 --- a/lib/spack/spack/cmd/pkg.py +++ b/lib/spack/spack/cmd/pkg.py @@ -24,11 +24,13 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: add_parser = sp.add_parser("add", help=pkg_add.__doc__) arguments.add_common_arguments(add_parser, ["packages"]) + add_parser.set_defaults(subparser=add_parser) list_parser = sp.add_parser("list", help=pkg_list.__doc__) list_parser.add_argument( "rev", default="HEAD", nargs="?", help="revision to list packages for" ) + list_parser.set_defaults(subparser=list_parser) diff_parser = sp.add_parser("diff", help=pkg_diff.__doc__) diff_parser.add_argument( @@ -37,12 +39,14 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: diff_parser.add_argument( "rev2", nargs="?", default="HEAD", help="revision to compare to rev1 (default is HEAD)" ) + diff_parser.set_defaults(subparser=diff_parser) add_parser = sp.add_parser("added", help=pkg_added.__doc__) add_parser.add_argument("rev1", nargs="?", default="HEAD^", help="revision to compare against") add_parser.add_argument( "rev2", nargs="?", default="HEAD", help="revision to compare to rev1 (default is HEAD)" ) + add_parser.set_defaults(subparser=add_parser) add_parser = sp.add_parser("changed", help=pkg_changed.__doc__) add_parser.add_argument("rev1", nargs="?", default="HEAD^", help="revision to compare against") @@ -56,12 +60,14 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: default="C", help="types of changes to show (A: added, R: removed, C: changed); default is 'C'", ) + add_parser.set_defaults(subparser=add_parser) rm_parser = sp.add_parser("removed", help=pkg_removed.__doc__) rm_parser.add_argument("rev1", nargs="?", default="HEAD^", help="revision to compare against") rm_parser.add_argument( "rev2", nargs="?", default="HEAD", help="revision to compare to rev1 (default is HEAD)" ) + rm_parser.set_defaults(subparser=rm_parser) # explicitly add help for `spack pkg grep` with just `--help` and NOT `-h`. This is so # that the very commonly used -h (no filename) argument can be passed through to grep @@ -70,6 +76,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: "grep_args", nargs=argparse.REMAINDER, default=None, help="arguments for grep" ) grep_parser.add_argument("--help", action="help", help="show this help message and exit") + grep_parser.set_defaults(subparser=grep_parser) source_parser = sp.add_parser("source", help=pkg_source.__doc__) source_parser.add_argument( @@ -80,9 +87,11 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: help="dump canonical source as used by package hash", ) arguments.add_common_arguments(source_parser, ["spec"]) + source_parser.set_defaults(subparser=source_parser) hash_parser = sp.add_parser("hash", help=pkg_hash.__doc__) arguments.add_common_arguments(hash_parser, ["spec"]) + hash_parser.set_defaults(subparser=hash_parser) def pkg_add(args): @@ -138,7 +147,7 @@ def pkg_source(args): """dump source code for a package""" specs = spack.cmd.parse_specs(args.spec, concretize=False) if len(specs) != 1: - tty.die("spack pkg source requires exactly one spec") + args.subparser.error("requires exactly one spec") spec = specs[0] filename = spack.repo.PATH.filename_for_package_name(spec.name) @@ -252,6 +261,6 @@ def pkg(parser, args, unknown_args): if args.pkg_command == "grep": return pkg_grep(args, unknown_args) elif unknown_args: - tty.die("unrecognized arguments: %s" % " ".join(unknown_args)) + args.subparser.error("unrecognized arguments: %s" % " ".join(unknown_args)) else: return action[args.pkg_command](args) diff --git a/lib/spack/spack/cmd/python.py b/lib/spack/spack/cmd/python.py index 1f99f0b9b199dc..d6a68fb152131c 100644 --- a/lib/spack/spack/cmd/python.py +++ b/lib/spack/spack/cmd/python.py @@ -71,11 +71,11 @@ def python(parser, args, unknown_args): return if unknown_args: - tty.die("Unknown arguments:", " ".join(unknown_args)) + args.subparser.error("unrecognized arguments: %s" % " ".join(unknown_args)) # Unexpected behavior from supplying both if args.python_command and args.python_args: - tty.die("You can only specify a command OR script, but not both.") + args.subparser.error("you can only specify a command OR script, but not both") # Ensure that spack.repo.PATH is initialized spack.repo.PATH.repos diff --git a/lib/spack/spack/cmd/remove.py b/lib/spack/spack/cmd/remove.py index a73077e62492d4..d15d1250c28b4c 100644 --- a/lib/spack/spack/cmd/remove.py +++ b/lib/spack/spack/cmd/remove.py @@ -31,7 +31,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: def remove(parser, args): - env = spack.cmd.require_active_env(cmd_name="remove") + env = spack.cmd.require_active_env(args.subparser) with env.write_transaction(): if args.all: diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 6525de9951a8f4..d38228f840762c 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -5,21 +5,29 @@ import argparse import os import shlex +import sys import tempfile from typing import Any, Dict, Generator, List, Optional, Tuple, Union import spack import spack.caches +import spack.ci import spack.config +import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.repo +import spack.spec import spack.util.executable import spack.util.git import spack.util.path +import spack.util.spack_json as sjson import spack.util.spack_yaml from spack.cmd.common import arguments from spack.error import SpackError from spack.llnl.util.tty import color +from spack.version import StandardVersion + +from . import doc_dedented, doc_first_line description = "manage package source repositories" section = "config" @@ -30,7 +38,9 @@ def setup_parser(subparser: argparse.ArgumentParser): sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="repo_command") # Create - create_parser = sp.add_parser("create", help=repo_create.__doc__) + create_parser = sp.add_parser( + "create", description=doc_dedented(repo_create), help=doc_first_line(repo_create) + ) create_parser.add_argument("directory", help="directory to create the repo in") create_parser.add_argument( "namespace", help="name or namespace to identify packages in the repository" @@ -46,7 +56,9 @@ def setup_parser(subparser: argparse.ArgumentParser): ) # List - list_parser = sp.add_parser("list", aliases=["ls"], help=repo_list.__doc__) + list_parser = sp.add_parser( + "list", aliases=["ls"], description=doc_dedented(repo_list), help=doc_first_line(repo_list) + ) list_parser.add_argument( "--scope", action=arguments.ConfigScope, @@ -58,9 +70,14 @@ def setup_parser(subparser: argparse.ArgumentParser): output_group.add_argument( "--namespaces", action="store_true", help="show repository namespaces only" ) + output_group.add_argument( + "--json", action="store_true", help="output repositories as machine-readable json records" + ) # Add - add_parser = sp.add_parser("add", help=repo_add.__doc__) + add_parser = sp.add_parser( + "add", description=doc_dedented(repo_add), help=doc_first_line(repo_add) + ) add_parser.add_argument( "path_or_repo", help="path or git repository of a Spack package repository" ) @@ -91,7 +108,9 @@ def setup_parser(subparser: argparse.ArgumentParser): ) # Set (modify existing repository configuration) - set_parser = sp.add_parser("set", help=repo_set.__doc__) + set_parser = sp.add_parser( + "set", description=doc_dedented(repo_set), help=doc_first_line(repo_set) + ) set_parser.add_argument("namespace", help="namespace of a Spack package repository") set_parser.add_argument( "--destination", help="destination to clone git repository into", action="store" @@ -111,7 +130,12 @@ def setup_parser(subparser: argparse.ArgumentParser): ) # Remove - remove_parser = sp.add_parser("remove", help=repo_remove.__doc__, aliases=["rm"]) + remove_parser = sp.add_parser( + "remove", + description=doc_dedented(repo_remove), + help=doc_first_line(repo_remove), + aliases=["rm"], + ) remove_parser.add_argument( "namespace_or_path", help="namespace or path of a Spack package repository" ) @@ -126,7 +150,9 @@ def setup_parser(subparser: argparse.ArgumentParser): ) # Migrate - migrate_parser = sp.add_parser("migrate", help=repo_migrate.__doc__) + migrate_parser = sp.add_parser( + "migrate", description=doc_dedented(repo_migrate), help=doc_first_line(repo_migrate) + ) migrate_parser.add_argument( "namespace_or_path", help="path to a Spack package repository directory" ) @@ -143,7 +169,9 @@ def setup_parser(subparser: argparse.ArgumentParser): ) # Update - update_parser = sp.add_parser("update", help=repo_update.__doc__) + update_parser = sp.add_parser( + "update", description=doc_dedented(repo_update), help=doc_first_line(repo_update) + ) update_parser.add_argument("names", nargs="*", default=[], help="repositories to update") update_parser.add_argument( "--remote", @@ -167,6 +195,27 @@ def setup_parser(subparser: argparse.ArgumentParser): "--commit", "-c", nargs="?", default=None, help="name of a commit to change to" ) + # Show updates + show_version_updates_parser = sp.add_parser( + "show-version-updates", help=repo_show_version_updates.__doc__ + ) + show_version_updates_parser.add_argument( + "--no-manual-packages", action="store_true", help="exclude manual packages" + ) + show_version_updates_parser.add_argument( + "--no-git-versions", action="store_true", help="exclude versions from git" + ) + show_version_updates_parser.add_argument( + "--only-redistributable", action="store_true", help="exclude non-redistributable packages" + ) + show_version_updates_parser.add_argument( + "repository", help="name or path of the repository to analyze" + ) + show_version_updates_parser.add_argument( + "from_ref", help="git ref from which to start looking at changes" + ) + show_version_updates_parser.add_argument("to_ref", help="git ref to end looking at changes") + def repo_create(args): """create a new package repository""" @@ -258,7 +307,7 @@ def repo_add(args): def repo_remove(args): """remove a repository from Spack's configuration""" - scopes = [args.scope] if args.scope else list(spack.config.CONFIG.scopes.keys()) + scopes = [args.scope] if args.scope else reversed(list(spack.config.CONFIG.scopes.keys())) found_and_removed = False for scope in scopes: found_and_removed |= _remove_repo(args.namespace_or_path, scope) @@ -283,7 +332,7 @@ def _remove_repo(namespace_or_path, scope): for name, descriptor in descriptors.items(): descriptor.initialize(fetch=False) - # For now you cannot delete monorepos with multipe package repositories from config, + # For now you cannot delete monorepos with multiple package repositories from config, # hence "all" and not "any". We can improve this later if needed. if all( r.namespace == namespace_or_path or r.root == canon_path @@ -302,7 +351,11 @@ def _remove_repo(namespace_or_path, scope): def repo_list(args): - """show registered repositories and their namespaces""" + """show registered repositories and their namespaces + + List all package repositories known to Spack. Repositories + can be local directories or remote git repositories. + """ descriptors = spack.repo.RepoDescriptors.from_config( lock=spack.repo.package_repository_lock(), config=spack.config.CONFIG, scope=args.scope ) @@ -320,25 +373,61 @@ def repo_list(args): print(maybe_repo.namespace) return - # Default table format: collect all repository information for aligned output + # Collect all repository information repo_info = [] for name, path, maybe_repo in _iter_repos_from_descriptors(descriptors): if isinstance(maybe_repo, spack.repo.Repo): - repo_info.append( - ("@g{[+]}", maybe_repo.namespace, maybe_repo.package_api_str, maybe_repo.root) - ) + status = "installed" + namespace = maybe_repo.namespace + api = maybe_repo.package_api_str + repo_path = maybe_repo.root elif maybe_repo is None: # Uninitialized Git-based repo case - repo_info.append(("@K{ - }", name, "", path)) + status = "uninitialized" + namespace = name + api = "" + repo_path = path else: # Exception/error case - repo_info.append(("@r{[-]}", name, "", f"{path}: {maybe_repo}")) + status = "error" + namespace = name + api = "" + repo_path = path + + # Add the repo info to our list + repo_info.append( + { + "name": name, + "namespace": namespace, + "path": repo_path, + "api_version": api, + "status": status, + "error": str(maybe_repo) if isinstance(maybe_repo, Exception) else None, + } + ) + + # Output in JSON format if requested + if args.json: + sjson.dump(repo_info, sys.stdout) + return + + # Default table format with aligned output + formatted_repo_info = [] + for repo in repo_info: + if repo["status"] == "installed": + status = "@g{[+]}" + elif repo["status"] == "uninitialized": + status = "@K{ - }" + else: # error + status = "@r{[-]}" - if repo_info: - max_namespace_width = max(len(namespace) for _, namespace, _, _ in repo_info) + 3 - max_api_width = max(len(api) for _, _, api, _ in repo_info) + 3 + formatted_repo_info.append((status, repo["namespace"], repo["api_version"], repo["path"])) + + if formatted_repo_info: + max_namespace_width = max(len(namespace) for _, namespace, _, _ in formatted_repo_info) + 3 + max_api_width = max(len(api) for _, _, api, _ in formatted_repo_info) + 3 # Print aligned output - for status, namespace, api, path in repo_info: + for status, namespace, api, path in formatted_repo_info: cpath = color.cescape(path) color.cprint( f"{status} {namespace:<{max_namespace_width}} {api:<{max_api_width}} {cpath}" @@ -346,7 +435,7 @@ def repo_list(args): def _get_repo(name_or_path: str) -> Optional[spack.repo.Repo]: - """Get a repo by path or namespace""" + """get a repo by path or namespace""" try: return spack.repo.from_path(name_or_path) except spack.repo.RepoError: @@ -490,7 +579,7 @@ def _iter_repos_from_descriptors( yield name, descriptor.repository, None # None indicates remote descriptor -def repo_update(args: Any) -> int: +def repo_update(args): """update one or more package repositories""" descriptors = spack.repo.RepoDescriptors.from_config( spack.repo.package_repository_lock(), spack.config.CONFIG @@ -560,12 +649,87 @@ def repo_update(args: Any) -> int: ) else: - tty.msg(f"{name}: Updated sucessfully.") + tty.msg(f"{name}: Updated successfully.") if active_flag: spack.config.set("repos", scope_repos, args.scope) - return 0 + +def repo_show_version_updates(args): + """show version specs that were added between two commits""" + # Get the repository by name or path + repo = _get_repo(args.repository) + + if repo is None: + tty.die(f"No such repository: {args.repository}") + + # Get packages that were changed or added between the refs + pkgs = spack.repo.get_all_package_diffs("AC", repo, args.from_ref, args.to_ref) + + # Filter out manual packages if requested + if args.no_manual_packages: + pkgs = { + pkg_name + for pkg_name in pkgs + if not spack.repo.PATH.get_pkg_class(pkg_name).manual_download + } + + if not pkgs: + tty.info("No packages were added or changed between the specified refs", stream=sys.stderr) + return 0 + + # Collect version specs that were added + specs_to_output = [] + + for pkg_name in pkgs: + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) + path = spack.repo.PATH.package_path(pkg_name) + + # Get all versions with checksums or commits + version_to_checksum: Dict[StandardVersion, str] = {} + for version in pkg_cls.versions: + version_dict = pkg_cls.versions[version] + if "sha256" in version_dict: + version_to_checksum[version] = version_dict["sha256"] + elif "commit" in version_dict: + version_to_checksum[version] = version_dict["commit"] + + # Find versions added between the refs + with fs.working_dir(os.path.dirname(path)): + added_checksums = spack.ci.filter_added_checksums( + version_to_checksum.values(), path, from_ref=args.from_ref, to_ref=args.to_ref + ) + new_versions = [v for v, c in version_to_checksum.items() if c in added_checksums] + + # Create specs for new versions + for version in new_versions: + version_spec = spack.spec.Spec(pkg_name) + version_spec.constrain(f"@={version}") + specs_to_output.append(version_spec) + + # Filter out git versions if requested + if args.no_git_versions: + specs_to_output = [ + spec + for spec in specs_to_output + if "commit" not in spack.repo.PATH.get_pkg_class(spec.name).versions[spec.version] + ] + + # Filter out non-redistributable packages if requested + if args.only_redistributable: + specs_to_output = [ + spec + for spec in specs_to_output + if spack.repo.PATH.get_pkg_class(spec.name).redistribute_source(spec) + ] + + if not specs_to_output: + tty.info("No new package versions found between the specified refs", stream=sys.stderr) + return 0 + + # Output specs one per line + for spec in specs_to_output: + print(spec) def repo(parser, args): @@ -579,4 +743,5 @@ def repo(parser, args): "rm": repo_remove, "migrate": repo_migrate, "update": repo_update, + "show-version-updates": repo_show_version_updates, }[args.repo_command](args) diff --git a/lib/spack/spack/cmd/resource.py b/lib/spack/spack/cmd/resource.py index 1792b7e3e9f99b..657d1789a80cf4 100644 --- a/lib/spack/spack/cmd/resource.py +++ b/lib/spack/spack/cmd/resource.py @@ -28,7 +28,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: def _show_patch(sha256): """Show a record from the patch index.""" - patches = spack.repo.PATH.patch_index.index + patches = spack.repo.PATH.get_patch_index().index data = patches.get(sha256) if not data: @@ -59,7 +59,7 @@ def _show_patch(sha256): def resource_list(args): """list all resources known to spack (currently just patches)""" - patches = spack.repo.PATH.patch_index.index + patches = spack.repo.PATH.get_patch_index().index for sha256 in patches: if args.only_hashes: print(sha256) diff --git a/lib/spack/spack/cmd/restage.py b/lib/spack/spack/cmd/restage.py index 5337f39f285929..0d53e581eea48c 100644 --- a/lib/spack/spack/cmd/restage.py +++ b/lib/spack/spack/cmd/restage.py @@ -5,7 +5,6 @@ import argparse import spack.cmd -import spack.llnl.util.tty as tty from spack.cmd.common import arguments description = "revert checked out package source code" @@ -19,7 +18,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: def restage(parser, args): if not args.specs: - tty.die("spack restage requires at least one package spec.") + args.subparser.error("requires at least one package spec") specs = spack.cmd.parse_specs(args.specs, concretize=True) for spec in specs: diff --git a/lib/spack/spack/cmd/solve.py b/lib/spack/spack/cmd/solve.py index 4063f1c64ea109..e43c21e46e119c 100644 --- a/lib/spack/spack/cmd/solve.py +++ b/lib/spack/spack/cmd/solve.py @@ -148,7 +148,7 @@ def solve(parser, args): elif env: specs = list(env.user_specs) else: - tty.die("spack solve requires at least one spec or an active environment") + args.subparser.error("requires at least one spec or an active environment") solver = asp.Solver() output = sys.stdout if "asp" in show else None diff --git a/lib/spack/spack/cmd/spec.py b/lib/spack/spack/cmd/spec.py index aeedb3e37a3b78..d59cc010253e76 100644 --- a/lib/spack/spack/cmd/spec.py +++ b/lib/spack/spack/cmd/spec.py @@ -10,7 +10,6 @@ import spack.environment as ev import spack.hash_types as ht import spack.llnl.util.lang as lang -import spack.llnl.util.tty as tty import spack.package_base import spack.spec import spack.store @@ -98,7 +97,7 @@ def spec(parser, args): env.concretize() concrete_specs = env.concrete_roots() else: - tty.die("spack spec requires at least one spec or an active environment") + args.subparser.error("requires at least one spec or an active environment") # With --yaml, --json, or --format, just print the raw specs to output if args.format: diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index a12d6ca11700d6..b65b5c71bef51d 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -73,7 +73,7 @@ def stage(parser, args): if not args.specs: env = ev.active_environment() if not env: - tty.die("`spack stage` requires a spec or an active environment") + args.subparser.error("requires a spec or an active environment") return _stage_env(env, filter) specs = spack.cmd.parse_specs(args.specs, concretize=False) @@ -84,7 +84,7 @@ def stage(parser, args): # prevent multiple specs from extracting in the same folder if len(specs) > 1 and custom_path: - tty.die("`--path` requires a single spec, but multiple were provided") + args.subparser.error("--path requires a single spec, but multiple were provided") specs = spack.cmd.matching_specs_from_env(specs) for spec in specs: @@ -104,7 +104,6 @@ def stage(parser, args): def _stage_env(env: ev.Environment, filter): tty.msg(f"Staging specs from environment {env.name}") for spec in spack.traverse.traverse_nodes(env.concrete_roots()): - if filter(spec): continue diff --git a/lib/spack/spack/cmd/style.py b/lib/spack/spack/cmd/style.py index 49111afc1d145f..a9e3e20bf14904 100644 --- a/lib/spack/spack/cmd/style.py +++ b/lib/spack/spack/cmd/style.py @@ -6,42 +6,33 @@ import os import re import sys -import warnings -from itertools import zip_longest -from typing import Callable, Dict, List, Optional, Set +from pathlib import Path +from typing import Dict, List, Optional, Set, Union import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as color import spack.paths import spack.repo import spack.util.git -import spack.util.spack_yaml +from spack.cmd.common.spec_strings import ( + _check_spec_strings, + _spec_str_default_handler, + _spec_str_fix_handler, +) from spack.llnl.util.filesystem import working_dir -from spack.spec_parser import NAME, VERSION_LIST, SpecTokens -from spack.tokenize import Token, TokenBase, Tokenizer from spack.util.executable import Executable, which description = "runs source code style checks on spack" section = "developer" level = "long" - -def grouper(iterable, n, fillvalue=None): - """Collect data into fixed-length chunks or blocks""" - # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx" - args = [iter(iterable)] * n - for group in zip_longest(*args, fillvalue=fillvalue): - yield filter(None, group) - - #: List of paths to exclude from checks -- relative to spack root exclude_paths = [os.path.relpath(spack.paths.vendor_path, spack.paths.prefix)] -#: Order in which tools should be run. flake8 is last so that it can -#: double-check the results of other tools (if, e.g., ``--fix`` was provided) +#: Order in which tools should be run. #: The list maps an executable name to a method to ensure the tool is #: bootstrapped or present in the environment. -tool_names = ["import", "isort", "black", "flake8", "mypy"] +tool_names = ["import", "ruff-format", "ruff-check", "mypy"] #: warnings to ignore in mypy mypy_ignores = [ @@ -51,22 +42,15 @@ def grouper(iterable, n, fillvalue=None): ] -def is_package(f): - """Whether flake8 should consider a file as a core file or a package. - - We run flake8 with different exceptions for the core and for - packages, since we allow ``from spack.package import *`` and poking globals - into packages. - """ - return f.startswith("var/spack/") and f.endswith("package.py") - - #: decorator for adding tools to the list class tool: - def __init__(self, name: str, required: bool = False, external: bool = True) -> None: + def __init__( + self, name: str, cmd: Optional[str] = None, required: bool = False, external: bool = True + ) -> None: self.name = name self.external = external self.required = required + self.cmd = cmd if cmd else name def __call__(self, fun): self.fun = fun @@ -75,18 +59,18 @@ def __call__(self, fun): @property def installed(self) -> bool: - return bool(which(self.name)) if self.external else True + return bool(which(self.cmd)) if self.external else True @property def executable(self) -> Optional[Executable]: - return which(self.name) if self.external else None + return which(self.cmd) if self.external else None #: tools we run in spack style tools: Dict[str, tool] = {} -def changed_files(base="develop", untracked=True, all_files=False, root=None): +def changed_files(base="develop", untracked=True, all_files=False, root=None) -> List[Path]: """Get list of changed files in the Spack repository. Arguments: @@ -145,7 +129,7 @@ def changed_files(base="develop", untracked=True, all_files=False, root=None): if any(os.path.realpath(f).startswith(e) for e in excludes): continue - changed.add(f) + changed.add(Path(f)) return sorted(changed) @@ -159,7 +143,10 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: help="branch to compare against to determine changed files (default: develop)", ) subparser.add_argument( - "-a", "--all", action="store_true", help="check all files, not just changed files" + "-a", + "--all", + action="store_true", + help="check all files, not just changed files (applies only to Import Check)", ) subparser.add_argument( "-r", @@ -212,21 +199,27 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument("files", nargs=argparse.REMAINDER, help="specific files to check") -def cwd_relative(path, root, initial_working_dir): +def cwd_relative(path: Path, root: Union[Path, str], initial_working_dir: Path) -> Path: """Translate prefix-relative path to current working directory-relative.""" - return os.path.relpath(os.path.join(root, path), initial_working_dir) + if path.is_absolute(): + return path + return Path(os.path.relpath((root / path), initial_working_dir)) def rewrite_and_print_output( - output, args, re_obj=re.compile(r"^(.+):([0-9]+):"), replacement=r"{0}:{1}:" + output, + root, + working_dir, + root_relative, + re_obj=re.compile(r"^(.+):([0-9]+):"), + replacement=r"{0}:{1}:", ): - """rewrite ouput with :: format to respect path args""" + """rewrite output with :: format to respect path args""" # print results relative to current working directory def translate(match): return replacement.format( - cwd_relative(match.group(1), args.root, args.initial_working_dir), - *list(match.groups()[1:]), + cwd_relative(Path(match.group(1)), root, working_dir), *list(match.groups()[1:]) ) for line in output.split("\n"): @@ -236,28 +229,11 @@ def translate(match): # some mypy annotations can't be disabled in older mypys (e.g. .971, which # is the only mypy that supports python 3.6), so we filter them here. continue - if not args.root_relative and re_obj: + if not root_relative and re_obj: line = re_obj.sub(translate, line) print(line) -def print_style_header(file_list, args, tools_to_run): - tty.msg("Running style checks on spack", "selected: " + ", ".join(tools_to_run)) - # translate modified paths to cwd_relative if needed - paths = [filename.strip() for filename in file_list] - if not args.root_relative: - paths = [cwd_relative(filename, args.root, args.initial_working_dir) for filename in paths] - - tty.msg("Modified files", *paths) - sys.stdout.flush() - - -def print_tool_header(tool): - sys.stdout.flush() - tty.msg("Running %s checks" % tool) - sys.stdout.flush() - - def print_tool_result(tool, returncode): if returncode == 0: color.cprint(" @g{%s checks were clean}" % tool) @@ -265,30 +241,65 @@ def print_tool_result(tool, returncode): color.cprint(" @r{%s found errors}" % tool) -@tool("flake8", required=True) -def run_flake8(flake8_cmd, file_list, args): - returncode = 0 - output = "" - # run in chunks of 100 at a time to avoid line length limit - # filename parameter in config *does not work* for this reliably - for chunk in grouper(file_list, 100): - output = flake8_cmd( - # always run with config from running spack prefix - "--config=%s" % os.path.join(spack.paths.prefix, ".flake8"), - *chunk, - fail_on_error=False, - output=str, - ) - returncode |= flake8_cmd.returncode +@tool("ruff-check", cmd="ruff") +def ruff_check(file_list, args): + """Run the ruff-check command. Handles config and non generic ruff argument logic""" + cmd_args = ["--config", os.path.join(spack.paths.prefix, "pyproject.toml"), "--quiet"] + if args.fix: + cmd_args += ["--fix", "--no-unsafe-fixes"] + else: + cmd_args += ["--no-fix"] + return run_ruff( + file_list, "check", cmd_args, args.root, args.initial_working_dir, args.root_relative + ) - rewrite_and_print_output(output, args) - print_tool_result("flake8", returncode) +@tool("ruff-format", cmd="ruff") +def ruff_format(file_list, args): + """Run the ruff format command""" + cmd_args = ["--config", os.path.join(spack.paths.prefix, "pyproject.toml"), "--quiet"] + if not args.fix: + cmd_args += ["--check", "--diff"] + return run_ruff( + file_list, "format", cmd_args, args.root, args.initial_working_dir, args.root_relative + ) + + +def run_ruff( + file_list: List[Path], + cmd: str, + args: List[str], + root: Path, + working_dir: Path, + root_relative: bool, +): + """Run the ruff tool""" + ruff_cmd = tools[f"ruff-{cmd}"].executable + if not ruff_cmd: + tty.warn("Cannot execute requested tool: ruff\nCannot find tool") + return -1 + + files = (str(x) for x in file_list) + if color.get_color_when(): + args += ("--color", "auto") + pat = re.compile("would reformat +(.*)") + replacement = "would reformat {0}" + + packed_args = (cmd,) + (*args,) + tuple(files) + output = ruff_cmd(*packed_args, fail_on_error=False, output=str, error=str) + returncode = ruff_cmd.returncode + rewrite_and_print_output(output, root, working_dir, root_relative, pat, replacement) + + print_tool_result(f"ruff-{cmd}", returncode) return returncode @tool("mypy") -def run_mypy(mypy_cmd, file_list, args): +def run_mypy(file_list, args): + mypy_cmd = tools["mypy"].executable + if not mypy_cmd: + tty.warn("Cannot execute requested tool: mypy\nCannot find tool") + return -1 # always run with config from running spack prefix common_mypy_args = [ "--config-file", @@ -306,80 +317,26 @@ def run_mypy(mypy_cmd, file_list, args): output = mypy_cmd(*mypy_args, fail_on_error=False, output=str) returncode |= mypy_cmd.returncode - rewrite_and_print_output(output, args) + rewrite_and_print_output(output, args.root, args.initial_working_dir, args.root_relative) print_tool_result("mypy", returncode) return returncode -@tool("isort") -def run_isort(isort_cmd, file_list, args): - # always run with config from running spack prefix - isort_args = ("--settings-path", os.path.join(spack.paths.prefix, "pyproject.toml")) - if not args.fix: - isort_args += ("--check", "--diff") - - pat = re.compile("ERROR: (.*) Imports are incorrectly sorted") - replacement = "ERROR: {0} Imports are incorrectly sorted" - returncode = [0] - - def process_files(file_list, is_args): - for chunk in grouper(file_list, 100): - packed_args = is_args + tuple(chunk) - output = isort_cmd(*packed_args, fail_on_error=False, output=str, error=str) - returncode[0] |= isort_cmd.returncode - - rewrite_and_print_output(output, args, pat, replacement) - - # packages - process_files(filter(is_package, file_list), isort_args) - # non-packages - process_files(filter(lambda f: not is_package(f), file_list), isort_args) - - print_tool_result("isort", returncode[0]) - return returncode[0] - - -@tool("black") -def run_black(black_cmd, file_list, args): - # always run with config from running spack prefix - black_args = ("--config", os.path.join(spack.paths.prefix, "pyproject.toml")) - if not args.fix: - black_args += ("--check", "--diff") - if color.get_color_when(): # only show color when spack would - black_args += ("--color",) - - pat = re.compile("would reformat +(.*)") - replacement = "would reformat {0}" - returncode = 0 - output = "" - # run in chunks of 100 at a time to avoid line length limit - # filename parameter in config *does not work* for this reliably - for chunk in grouper(file_list, 100): - packed_args = black_args + tuple(chunk) - output = black_cmd(*packed_args, fail_on_error=False, output=str, error=str) - returncode |= black_cmd.returncode - rewrite_and_print_output(output, args, pat, replacement) - - print_tool_result("black", returncode) - - return returncode - - -def _module_part(root: str, expr: str): +def _module_part(root: Path, expr: str): parts = expr.split(".") # spack.pkg is for repositories, don't try to resolve it here. if expr.startswith(spack.repo.PKG_MODULE_PREFIX_V1) or expr == "spack.pkg": return None while parts: - f1 = os.path.join(root, "lib", "spack", *parts) + ".py" - f2 = os.path.join(root, "lib", "spack", *parts, "__init__.py") + f1 = (root / "lib" / "spack").joinpath(*parts).with_suffix(".py") + f2 = (root / "lib" / "spack").joinpath(*parts, "__init__.py") if ( - os.path.exists(f1) + f1.exists() # ensure case sensitive match - and f"{parts[-1]}.py" in os.listdir(os.path.dirname(f1)) - or os.path.exists(f2) + and any(p.name == f"{parts[-1]}.py" for p in f1.parent.iterdir()) + or f2.exists() ): return ".".join(parts) parts.pop() @@ -387,13 +344,15 @@ def _module_part(root: str, expr: str): def _run_import_check( - file_list: List[str], + file_list: List[Path], *, fix: bool, root_relative: bool, - root=spack.paths.prefix, - working_dir=spack.paths.prefix, + root: Path, + working_dir: Path, out=sys.stdout, + base="develop", + all=False, ): if sys.version_info < (3, 9): print("import check requires Python 3.9 or later") @@ -402,8 +361,8 @@ def _run_import_check( is_use = re.compile(r"(? None: - # only move the compiler to the back if it exists and is not already at the end - if not 0 <= idx < len(blocks) - 1: - return - # if there's only whitespace after the compiler, don't move it - if all(token.kind == _LegacySpecTokens.WS for block in blocks[idx + 1 :] for token in block): - return - # rotate left and always add at least one WS token between compiler and previous token - compiler_block = blocks.pop(idx) - if compiler_block[0].kind != _LegacySpecTokens.WS: - compiler_block.insert(0, Token(_LegacySpecTokens.WS, " ")) - # delete the WS tokens from the new first block if it was at the very start, to prevent leading - # WS tokens. - while idx == 0 and blocks[0][0].kind == _LegacySpecTokens.WS: - blocks[0].pop(0) - blocks.append(compiler_block) - - -def _spec_str_format(spec_str: str) -> Optional[str]: - """Given any string, try to parse as spec string, and rotate the compiler token to the end - of each spec instance. Returns the formatted string if it was changed, otherwise None.""" - # We parse blocks of tokens that include leading whitespace, and move the compiler block to - # the end when we hit a dependency ^... or the end of a string. - # [@3.1][ +foo][ +bar][ %gcc@3.1][ +baz] - # [@3.1][ +foo][ +bar][ +baz][ %gcc@3.1] - - current_block: List[Token] = [] - blocks: List[List[Token]] = [] - compiler_block_idx = -1 - in_edge_attr = False - - legacy_tokenizer = Tokenizer(_LegacySpecTokens) - - for token in legacy_tokenizer.tokenize(spec_str): - if token.kind == _LegacySpecTokens.UNEXPECTED: - # parsing error, we cannot fix this string. - return None - elif token.kind in (_LegacySpecTokens.COMPILER, _LegacySpecTokens.COMPILER_AND_VERSION): - # multiple compilers are not supported in Spack v0.x, so early return - if compiler_block_idx != -1: - return None - current_block.append(token) - blocks.append(current_block) - current_block = [] - compiler_block_idx = len(blocks) - 1 - elif token.kind in ( - _LegacySpecTokens.START_EDGE_PROPERTIES, - _LegacySpecTokens.DEPENDENCY, - _LegacySpecTokens.UNQUALIFIED_PACKAGE_NAME, - _LegacySpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, - ): - _spec_str_reorder_compiler(compiler_block_idx, blocks) - compiler_block_idx = -1 - if token.kind == _LegacySpecTokens.START_EDGE_PROPERTIES: - in_edge_attr = True - current_block.append(token) - blocks.append(current_block) - current_block = [] - elif token.kind == _LegacySpecTokens.END_EDGE_PROPERTIES: - in_edge_attr = False - current_block.append(token) - blocks.append(current_block) - current_block = [] - elif in_edge_attr: - current_block.append(token) - elif token.kind in ( - _LegacySpecTokens.VERSION_HASH_PAIR, - _LegacySpecTokens.GIT_VERSION, - _LegacySpecTokens.VERSION, - _LegacySpecTokens.PROPAGATED_BOOL_VARIANT, - _LegacySpecTokens.BOOL_VARIANT, - _LegacySpecTokens.PROPAGATED_KEY_VALUE_PAIR, - _LegacySpecTokens.KEY_VALUE_PAIR, - _LegacySpecTokens.DAG_HASH, - ): - current_block.append(token) - blocks.append(current_block) - current_block = [] - elif token.kind == _LegacySpecTokens.WS: - current_block.append(token) - else: - raise ValueError(f"unexpected token {token}") - - if current_block: - blocks.append(current_block) - _spec_str_reorder_compiler(compiler_block_idx, blocks) - - new_spec_str = "".join(token.value for block in blocks for token in block) - return new_spec_str if spec_str != new_spec_str else None - - -SpecStrHandler = Callable[[str, int, int, str, str], None] - - -def _spec_str_default_handler(path: str, line: int, col: int, old: str, new: str): - """A SpecStrHandler that prints formatted spec strings and their locations.""" - print(f"{path}:{line}:{col}: `{old}` -> `{new}`") - - -def _spec_str_fix_handler(path: str, line: int, col: int, old: str, new: str): - """A SpecStrHandler that updates formatted spec strings in files.""" - with open(path, "r", encoding="utf-8") as f: - lines = f.readlines() - new_line = lines[line - 1].replace(old, new) - if new_line == lines[line - 1]: - tty.warn(f"{path}:{line}:{col}: could not apply fix: `{old}` -> `{new}`") - return - lines[line - 1] = new_line - print(f"{path}:{line}:{col}: fixed `{old}` -> `{new}`") - with open(path, "w", encoding="utf-8") as f: - f.writelines(lines) - - -def _spec_str_ast(path: str, tree: ast.AST, handler: SpecStrHandler) -> None: - """Walk the AST of a Python file and apply handler to formatted spec strings.""" - for node in ast.walk(tree): - if sys.version_info >= (3, 8): - if isinstance(node, ast.Constant) and isinstance(node.value, str): - current_str = node.value - else: - continue - elif isinstance(node, ast.Str): - current_str = node.s - else: - continue - if not IS_PROBABLY_COMPILER.search(current_str): - continue - new = _spec_str_format(current_str) - if new is not None: - handler(path, node.lineno, node.col_offset, current_str, new) - - -def _spec_str_json_and_yaml(path: str, data: dict, handler: SpecStrHandler) -> None: - """Walk a YAML or JSON data structure and apply handler to formatted spec strings.""" - queue = [data] - seen = set() - - while queue: - current = queue.pop(0) - if id(current) in seen: - continue - seen.add(id(current)) - if isinstance(current, dict): - queue.extend(current.values()) - queue.extend(current.keys()) - elif isinstance(current, list): - queue.extend(current) - elif isinstance(current, str) and IS_PROBABLY_COMPILER.search(current): - new = _spec_str_format(current) - if new is not None: - mark = getattr(current, "_start_mark", None) - if mark: - line, col = mark.line + 1, mark.column + 1 - else: - line, col = 0, 0 - handler(path, line, col, current, new) - - -def _check_spec_strings( - paths: List[str], handler: SpecStrHandler = _spec_str_default_handler -) -> None: - """Open Python, JSON and YAML files, and format their string literals that look like spec - strings. A handler is called for each formatting, which can be used to print or apply fixes.""" - for path in paths: - is_json_or_yaml = path.endswith(".json") or path.endswith(".yaml") or path.endswith(".yml") - is_python = path.endswith(".py") - if not is_json_or_yaml and not is_python: - continue - - try: - with open(path, "r", encoding="utf-8") as f: - # skip files that are likely too large to be user code or config - if os.fstat(f.fileno()).st_size > 1024 * 1024: - warnings.warn(f"skipping {path}: too large.") - continue - if is_json_or_yaml: - _spec_str_json_and_yaml(path, spack.util.spack_yaml.load_config(f), handler) - elif is_python: - _spec_str_ast(path, ast.parse(f.read()), handler) - except (OSError, spack.util.spack_yaml.SpackYAMLError, SyntaxError, ValueError): - warnings.warn(f"skipping {path}") - continue - - def style(parser, args): if args.spec_strings: if not args.files: @@ -757,26 +512,22 @@ def style(parser, args): return _check_spec_strings(args.files, handler) # save initial working directory for relativizing paths later - args.initial_working_dir = os.getcwd() + args.initial_working_dir = Path.cwd() # ensure that the config files we need actually exist in the spack prefix. # assertions b/c users should not ever see these errors -- they're checked in CI. - assert os.path.isfile(os.path.join(spack.paths.prefix, "pyproject.toml")) - assert os.path.isfile(os.path.join(spack.paths.prefix, ".flake8")) + assert (Path(spack.paths.prefix) / "pyproject.toml").is_file() # validate spack root if the user provided one - args.root = os.path.realpath(args.root) if args.root else spack.paths.prefix - spack_script = os.path.join(args.root, "bin", "spack") - if not os.path.exists(spack_script): + args.root = Path(args.root).resolve() if args.root else Path(spack.paths.prefix) + spack_script = args.root / "bin" / "spack" + if not spack_script.exists(): tty.die("This does not look like a valid spack root.", "No such file: '%s'" % spack_script) - file_list = args.files - if file_list: - - def prefix_relative(path): - return os.path.relpath(os.path.abspath(os.path.realpath(path)), args.root) + def prefix_relative(path: Union[Path, str]) -> Path: + return Path(os.path.relpath(os.path.abspath(os.path.realpath(path)), args.root)) - file_list = [prefix_relative(p) for p in file_list] + file_list = [prefix_relative(file) for file in args.files] # process --tool and --skip arguments selected = set(tool_names) @@ -794,16 +545,12 @@ def prefix_relative(path): _bootstrap_dev_dependencies() return_code = 0 - with working_dir(args.root): - if not file_list: - file_list = changed_files(args.base, args.untracked, args.all) - + with working_dir(str(args.root)): print_style_header(file_list, args, tools_to_run) for tool_name in tools_to_run: tool = tools[tool_name] - print_tool_header(tool_name) - return_code |= tool.fun(tool.executable, file_list, args) - + tty.msg(f"Running {tool.name} checks") + return_code |= tool.fun(file_list, args) if return_code == 0: tty.msg(color.colorize("@*{spack style checks were clean}")) else: diff --git a/lib/spack/spack/cmd/tags.py b/lib/spack/spack/cmd/tags.py index 39e6166c18a369..75694883be1cae 100644 --- a/lib/spack/spack/cmd/tags.py +++ b/lib/spack/spack/cmd/tags.py @@ -60,7 +60,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: def tags(parser, args): # Disallow combining all option with (positional) tags to avoid confusion if args.all and args.tag: - tty.die("Use the '--all' option OR provide tag(s) on the command line") + args.subparser.error("use the '--all' option OR provide tag(s) on the command line") # Provide a nice, simple message if database is empty if args.installed and not spack.environment.installed_specs(): @@ -93,7 +93,7 @@ def tags(parser, args): tag_pkgs = packages_with_tags(tags, args.installed, False) missing = "No installed packages" if args.installed else "None" for tag in sorted(tag_pkgs): - # TODO: Remove the sorting once we're sure noone has an old + # TODO: Remove the sorting once we're sure no one has an old # TODO: tag cache since it can accumulate duplicates. packages = sorted(list(set(tag_pkgs[tag]))) if isatty: diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index 3b2085e68a89a9..0c4736045dac4e 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -132,7 +132,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: "Test results will be filtered by space-" "separated suite name(s) and installed\nspecs when provided. " "If names are provided, then only results for those test\nsuites " - "will be shown. If installed specs are provided, then ony results" + "will be shown. If installed specs are provided, then only results" "\nmatching those specs will be shown." ) diff --git a/lib/spack/spack/cmd/undevelop.py b/lib/spack/spack/cmd/undevelop.py index 93b900b857430f..b78001debe1b8d 100644 --- a/lib/spack/spack/cmd/undevelop.py +++ b/lib/spack/spack/cmd/undevelop.py @@ -7,6 +7,7 @@ import spack.cmd import spack.config import spack.llnl.util.tty as tty +import spack.spec from spack.cmd.common import arguments description = "remove specs from an environment" @@ -32,7 +33,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: arguments.add_common_arguments(subparser, ["specs"]) -def _update_config(specs_to_remove, remove_all=False): +def _update_config(specs_to_remove): def change_fn(dev_config): modified = False for spec in specs_to_remove: @@ -40,38 +41,30 @@ def change_fn(dev_config): tty.msg("Undevelop: removing {0}".format(spec.name)) del dev_config[spec.name] modified = True - if remove_all and dev_config: - dev_config.clear() - modified = True return modified spack.config.update_all("develop", change_fn) def undevelop(parser, args): - remove_specs = None - remove_all = False + # TODO: when https://github.com/spack/spack/pull/35307 is merged, + # an active env is not required if a scope is specified + env = spack.cmd.require_active_env(args.subparser) + if args.all: - remove_all = True + remove_specs = [spack.spec.Spec(s) for s in env.dev_specs] else: remove_specs = spack.cmd.parse_specs(args.specs) - # TODO: when https://github.com/spack/spack/pull/35307 is merged, - # an active env is not required if a scope is specified - env = spack.cmd.require_active_env(cmd_name="undevelop") with env.write_transaction(): - _update_config(remove_specs, remove_all) + _update_config(remove_specs) if args.apply_changes: - for spec in remove_specs: - env.apply_develop(spec, path=None) + env.apply_develop(remove_specs, paths=None) updated_all_dev_specs = set(spack.config.get("develop")) - remove_spec_names = set(x.name for x in remove_specs) - if remove_all: - not_fully_removed = updated_all_dev_specs - else: - not_fully_removed = updated_all_dev_specs & remove_spec_names + remove_spec_names = set(x.name for x in remove_specs) + not_fully_removed = updated_all_dev_specs & remove_spec_names if not_fully_removed: tty.msg( diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index aadc4282801719..566d60534a797b 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -295,9 +295,9 @@ def uninstall_specs(args, specs): def uninstall(parser, args): if not args.specs and not args.all: - tty.die( - "uninstall requires at least one package argument.", - " Use `spack uninstall --all` to uninstall ALL packages.", + args.subparser.error( + "requires at least one package argument\n" + " use `spack uninstall --all` to uninstall ALL packages" ) # [None] here handles the --all case by forcing all specs to be returned diff --git a/lib/spack/spack/cmd/unload.py b/lib/spack/spack/cmd/unload.py index 3cc50cfba37e1c..50837e65105164 100644 --- a/lib/spack/spack/cmd/unload.py +++ b/lib/spack/spack/cmd/unload.py @@ -8,7 +8,6 @@ import spack.cmd import spack.cmd.common -import spack.error import spack.store import spack.user_environment as uenv from spack.cmd.common import arguments @@ -68,8 +67,8 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: def unload(parser, args): """unload spack packages from the user environment""" if args.specs and args.all: - raise spack.error.SpackError( - "Cannot specify specs on command line when unloading all specs with '--all'" + args.subparser.error( + "cannot specify specs on command line when unloading all specs with '--all'" ) hashes = os.environ.get(uenv.spack_loaded_hashes_var, "").split(os.pathsep) diff --git a/lib/spack/spack/cmd/verify.py b/lib/spack/spack/cmd/verify.py index 051757d9849ff6..13c1df05543dae 100644 --- a/lib/spack/spack/cmd/verify.py +++ b/lib/spack/spack/cmd/verify.py @@ -20,28 +20,26 @@ section = "admin" level = "long" -MANIFEST_SUBPARSER: Optional[argparse.ArgumentParser] = None - def setup_parser(subparser: argparse.ArgumentParser): - global MANIFEST_SUBPARSER sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="verify_command") - MANIFEST_SUBPARSER = sp.add_parser( + manifest_subparser = sp.add_parser( "manifest", help=verify_manifest.__doc__, description=verify_manifest.__doc__ ) - MANIFEST_SUBPARSER.add_argument( + manifest_subparser.set_defaults(subparser=manifest_subparser) + manifest_subparser.add_argument( "-l", "--local", action="store_true", help="verify only locally installed packages" ) - MANIFEST_SUBPARSER.add_argument( - "-j", "--json", action="store_true", help="ouptut json-formatted errors" + manifest_subparser.add_argument( + "-j", "--json", action="store_true", help="output json-formatted errors" ) - MANIFEST_SUBPARSER.add_argument("-a", "--all", action="store_true", help="verify all packages") - MANIFEST_SUBPARSER.add_argument( + manifest_subparser.add_argument("-a", "--all", action="store_true", help="verify all packages") + manifest_subparser.add_argument( "specs_or_files", nargs=argparse.REMAINDER, help="specs or files to verify" ) - manifest_sp_type = MANIFEST_SUBPARSER.add_mutually_exclusive_group() + manifest_sp_type = manifest_subparser.add_mutually_exclusive_group() manifest_sp_type.add_argument( "-s", "--specs", @@ -64,12 +62,14 @@ def setup_parser(subparser: argparse.ArgumentParser): libraries_subparser = sp.add_parser( "libraries", help=verify_libraries.__doc__, description=verify_libraries.__doc__ ) + libraries_subparser.set_defaults(subparser=libraries_subparser) arguments.add_common_arguments(libraries_subparser, ["constraint"]) versions_subparser = sp.add_parser( "versions", help=verify_versions.__doc__, description=verify_versions.__doc__ ) + versions_subparser.set_defaults(subparser=versions_subparser) arguments.add_common_arguments(versions_subparser, ["constraint"]) @@ -93,10 +93,7 @@ def verify_versions(args): 2. Installed package version not known by the package recipe 3. Installed package version deprecated in the package recipe """ - if args.specs: - specs = args.specs(installed=True) - else: - specs = spack.store.db.query(installed=True) + specs = args.specs(installed=True) msg_lines = _verify_version(specs) if msg_lines: @@ -189,7 +186,7 @@ def verify_manifest(args): if args.type == "files": if args.all: - MANIFEST_SUBPARSER.error("cannot use --all with --files") + args.subparser.error("cannot use --all with --files") for file in args.specs_or_files: results = spack.verify.check_file_manifest(file) @@ -220,7 +217,7 @@ def verify_manifest(args): env = ev.active_environment() specs = list(map(lambda x: spack.cmd.disambiguate_spec(x, env, local=local), spec_args)) else: - MANIFEST_SUBPARSER.error("use --all or specify specs to verify") + args.subparser.error("use --all or specify specs to verify") for spec in specs: tty.debug("Verifying package %s") diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index 406fa279fd9f82..e4c09e8f1f3ccc 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -32,6 +32,7 @@ YamlFilesystemView. """ + import argparse import sys diff --git a/lib/spack/spack/compilers/config.py b/lib/spack/spack/compilers/config.py index f34551180b42c9..244a7400404164 100644 --- a/lib/spack/spack/compilers/config.py +++ b/lib/spack/spack/compilers/config.py @@ -4,6 +4,7 @@ """This module contains functions related to finding compilers on the system, and configuring Spack to use multiple compilers. """ + import os import re import sys @@ -20,7 +21,7 @@ import spack.platforms import spack.repo import spack.spec -from spack.externals import ExternalSpecsParser, external_spec +from spack.externals import ExternalSpecsParser, external_spec, extract_dicts_from_configuration from spack.operating_systems import windows_os from spack.util.environment import get_path @@ -259,25 +260,24 @@ def from_packages_yaml( configuration: spack.config.Configuration, *, scope: Optional[str] = None ) -> List[spack.spec.Spec]: """Returns the compiler specs defined in the "packages" section of the configuration""" - externals_dicts = [] compiler_package_names = supported_compilers() - packages_yaml = configuration.get_config("packages", scope=scope) - for name, entry in packages_yaml.items(): - if name not in compiler_package_names: - continue + packages_yaml = configuration.deepcopy_as_builtin("packages", scope=scope) - externals_config = entry.get("externals", None) - if not externals_config: - continue + init_external_dicts = extract_dicts_from_configuration(packages_yaml) + init_external_dicts = list( + x + for x in init_external_dicts + if spack.spec.Spec(x["spec"]).name in compiler_package_names + ) - for current in externals_config: - # If extra_attributes is not there don't use this entry as a compiler. - if _EXTRA_ATTRIBUTES_KEY not in current: - header = f"The external spec '{current['spec']}' cannot be used as a compiler" - tty.debug(f"[{__file__}] {header}: missing the '{_EXTRA_ATTRIBUTES_KEY}' key") - continue + externals_dicts = [] + for current in init_external_dicts: + if _EXTRA_ATTRIBUTES_KEY not in current: + header = f"The external spec '{current['spec']}' cannot be used as a compiler" + tty.debug(f"[{__file__}] {header}: missing the '{_EXTRA_ATTRIBUTES_KEY}' key") + continue - externals_dicts.append(current) + externals_dicts.append(current) external_parser = ExternalSpecsParser(externals_dicts) return external_parser.all_specs() diff --git a/lib/spack/spack/compilers/libraries.py b/lib/spack/spack/compilers/libraries.py index d0831a602eb674..bac07a5b7e47f9 100644 --- a/lib/spack/spack/compilers/libraries.py +++ b/lib/spack/spack/compilers/libraries.py @@ -10,7 +10,7 @@ import stat import sys import tempfile -from typing import Dict, List, Optional, Set, Tuple +from typing import Dict, List, Optional, Set, Tuple, cast import spack.caches import spack.llnl.path @@ -380,7 +380,6 @@ class FileCompilerCache(CompilerCache): def __init__(self, cache: "FileCache") -> None: self.cache = cache - self.cache.init_entry(self.name) self._data: Dict[str, Dict[str, Optional[str]]] = {} def _get_entry(self, key: str, *, allow_empty: bool) -> Optional[CompilerCacheEntry]: @@ -395,13 +394,16 @@ def _get_entry(self, key: str, *, allow_empty: bool) -> Optional[CompilerCacheEn def get(self, compiler: spack.spec.Spec) -> CompilerCacheEntry: # Cache hit - try: - with self.cache.read_transaction(self.name) as f: - assert f is not None - self._data = json.loads(f.read()) - assert isinstance(self._data, dict) - except (json.JSONDecodeError, AssertionError): - self._data = {} + with self.cache.read_transaction(self.name) as f: + if f is not None: + try: + self._data = json.loads(f.read()) + if not isinstance(self._data, dict): + self._data = {} + except json.JSONDecodeError: + self._data = {} + else: + self._data = {} key = self._key(compiler) value = self._get_entry(key, allow_empty=False) @@ -410,11 +412,14 @@ def get(self, compiler: spack.spec.Spec) -> CompilerCacheEntry: # Cache miss with self.cache.write_transaction(self.name) as (old, new): - try: - assert old is not None - self._data = json.loads(old.read()) - assert isinstance(self._data, dict) - except (json.JSONDecodeError, AssertionError): + if old is not None: + try: + self._data = json.loads(old.read()) + if not isinstance(self._data, dict): + self._data = {} + except json.JSONDecodeError: + self._data = {} + else: self._data = {} # Use cache entry that may have been created by another process in the meantime. @@ -438,6 +443,4 @@ def _make_compiler_cache(): return FileCompilerCache(spack.caches.MISC_CACHE) -COMPILER_CACHE: CompilerCache = spack.llnl.util.lang.Singleton( # type: ignore - _make_compiler_cache -) +COMPILER_CACHE = cast(CompilerCache, spack.llnl.util.lang.Singleton(_make_compiler_cache)) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index ca473535f1ff89..8e10479d6ef78a 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -2,10 +2,11 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """High-level functions to concretize list of specs""" + import importlib import sys import time -from typing import Iterable, List, Optional, Sequence, Tuple, Union +from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Sequence, Tuple, Union import spack.compilers import spack.compilers.config @@ -20,9 +21,15 @@ SpecPair = Tuple[Spec, Spec] TestsType = Union[bool, Iterable[str]] +if TYPE_CHECKING: + from spack.solver.reuse import SpecFiltersFactory + def _concretize_specs_together( - abstract_specs: Sequence[Spec], tests: TestsType = False + abstract_specs: Sequence[Spec], + *, + tests: TestsType = False, + factory: Optional["SpecFiltersFactory"] = None, ) -> List[Spec]: """Given a number of specs as input, tries to concretize them together. @@ -30,16 +37,22 @@ def _concretize_specs_together( abstract_specs: abstract specs to be concretized tests: list of package names for which to consider tests dependencies. If True, all nodes will have test dependencies. If False, test dependencies will be disregarded. + factory: optional factory to produce a list of specs to be reused """ from spack.solver.asp import Solver allow_deprecated = spack.config.get("config:deprecated", False) - result = Solver().solve(abstract_specs, tests=tests, allow_deprecated=allow_deprecated) + result = Solver(specs_factory=factory).solve( + abstract_specs, tests=tests, allow_deprecated=allow_deprecated + ) return [s.copy() for s in result.specs] def concretize_together( - spec_list: Sequence[SpecPairInput], tests: TestsType = False + spec_list: Sequence[SpecPairInput], + *, + tests: TestsType = False, + factory: Optional["SpecFiltersFactory"] = None, ) -> List[SpecPair]: """Given a number of specs as input, tries to concretize them together. @@ -48,15 +61,19 @@ def concretize_together( already concrete spec or None if not yet concretized tests: list of package names for which to consider tests dependencies. If True, all nodes will have test dependencies. If False, test dependencies will be disregarded. + factory: optional factory to produce a list of specs to be reused """ to_concretize = [concrete if concrete else abstract for abstract, concrete in spec_list] abstract_specs = [abstract for abstract, _ in spec_list] - concrete_specs = _concretize_specs_together(to_concretize, tests=tests) + concrete_specs = _concretize_specs_together(to_concretize, tests=tests, factory=factory) return list(zip(abstract_specs, concrete_specs)) def concretize_together_when_possible( - spec_list: Sequence[SpecPairInput], tests: TestsType = False + spec_list: Sequence[SpecPairInput], + *, + tests: TestsType = False, + factory: Optional["SpecFiltersFactory"] = None, ) -> List[SpecPair]: """Given a number of specs as input, tries to concretize them together to the extent possible. @@ -68,6 +85,7 @@ def concretize_together_when_possible( already concrete spec or None if not yet concretized tests: list of package names for which to consider tests dependencies. If True, all nodes will have test dependencies. If False, test dependencies will be disregarded. + factory: optional factory to produce a list of specs to be reused """ from spack.solver.asp import Solver @@ -76,12 +94,25 @@ def concretize_together_when_possible( concrete: abstract for (abstract, concrete) in spec_list if concrete } - result_by_user_spec = {} + result_by_user_spec: Dict[Spec, Spec] = {} allow_deprecated = spack.config.get("config:deprecated", False) - for result in Solver().solve_in_rounds( + j = 0 + start = time.monotonic() + for result in Solver(specs_factory=factory).solve_in_rounds( to_concretize, tests=tests, allow_deprecated=allow_deprecated ): + now = time.monotonic() + duration = now - start + percentage = int((j + 1) / len(to_concretize) * 100) + for abstract, concrete in result.specs_by_input.items(): + tty.verbose( + f"{duration:6.1f}s [{percentage:3d}%] {concrete.cformat('{hash:7}')} " + f"{abstract.colored_str}" + ) + j += 1 + sys.stdout.flush() result_by_user_spec.update(result.specs_by_input) + start = now # If the "abstract" spec is a concrete spec from the previous concretization # translate it back to an abstract spec. Otherwise, keep the abstract spec @@ -92,7 +123,10 @@ def concretize_together_when_possible( def concretize_separately( - spec_list: Sequence[SpecPairInput], tests: TestsType = False + spec_list: Sequence[SpecPairInput], + *, + tests: TestsType = False, + factory: Optional["SpecFiltersFactory"] = None, ) -> List[SpecPair]: """Concretizes the input specs separately from each other. @@ -101,6 +135,7 @@ def concretize_separately( already concrete spec or None if not yet concretized tests: list of package names for which to consider tests dependencies. If True, all nodes will have test dependencies. If False, test dependencies will be disregarded. + factory: optional factory to produce a list of specs to be reused """ from spack.bootstrap import ( ensure_bootstrap_configuration, @@ -110,7 +145,7 @@ def concretize_separately( to_concretize = [abstract for abstract, concrete in spec_list if not concrete] args = [ - (i, str(abstract), tests) + (i, str(abstract), tests, factory) for i, abstract in enumerate(to_concretize) if not abstract.concrete ] @@ -156,13 +191,18 @@ def concretize_separately( for j, (i, concrete, duration) in enumerate( spack.util.parallel.imap_unordered( - _concretize_task, args, processes=num_procs, debug=tty.is_debug(), maxtaskperchild=1 + _concretize_task, + args, + processes=num_procs, + debug=tty.is_debug(), + maxtaskperchild=1, + serialize_env=True, ) ): ret.append((i, concrete)) - percentage = (j + 1) / len(args) * 100 + percentage = int((j + 1) / len(args) * 100) tty.verbose( - f"{duration:6.1f}s [{percentage:3.0f}%] {concrete.cformat('{hash:7}')} " + f"{duration:6.1f}s [{percentage:3d}%] {concrete.cformat('{hash:7}')} " f"{to_concretize[i].colored_str}" ) sys.stdout.flush() @@ -175,15 +215,22 @@ def concretize_separately( ] -def _concretize_task(packed_arguments: Tuple[int, str, TestsType]) -> Tuple[int, Spec, float]: - index, spec_str, tests = packed_arguments +def _concretize_task( + packed_arguments: Tuple[int, str, TestsType, Optional["SpecFiltersFactory"]], +) -> Tuple[int, Spec, float]: + index, spec_str, tests, factory = packed_arguments with tty.SuppressOutput(msg_enabled=False): start = time.time() - spec = concretize_one(Spec(spec_str), tests=tests) + spec = concretize_one(Spec(spec_str), tests=tests, factory=factory) return index, spec, time.time() - start -def concretize_one(spec: Union[str, Spec], tests: TestsType = False) -> Spec: +def concretize_one( + spec: Union[str, Spec], + *, + tests: TestsType = False, + factory: Optional["SpecFiltersFactory"] = None, +) -> Spec: """Return a concretized copy of the given spec. Args: @@ -206,7 +253,9 @@ def concretize_one(spec: Union[str, Spec], tests: TestsType = False) -> Spec: ) allow_deprecated = spack.config.get("config:deprecated", False) - result = Solver().solve([spec], tests=tests, allow_deprecated=allow_deprecated) + result = Solver(specs_factory=factory).solve( + [spec], tests=tests, allow_deprecated=allow_deprecated + ) # take the best answer opt, i, answer = min(result.answers) @@ -217,9 +266,9 @@ def concretize_one(spec: Union[str, Spec], tests: TestsType = False) -> Spec: name = providers[0] node = SpecBuilder.make_node(pkg=name) - assert ( - node in answer - ), f"cannot find {name} in the list of specs {','.join([n.pkg for n in answer.keys()])}" + assert node in answer, ( + f"cannot find {name} in the list of specs {','.join([n.pkg for n in answer.keys()])}" + ) concretized = answer[node] return concretized diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 8f14e958704ef1..85fcecfc81812b 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -26,6 +26,7 @@ schemas are in submodules of :py:mod:`spack.schema`. """ + import contextlib import copy import functools @@ -34,9 +35,10 @@ import pathlib import re import sys +import tempfile from collections import defaultdict from itertools import chain -from typing import Any, Callable, Dict, Generator, List, Optional, Set, Tuple, Union +from typing import Any, Callable, Dict, Generator, List, Optional, Set, Tuple, Union, cast from spack.vendor import jsonschema @@ -136,19 +138,15 @@ MAX_RECURSIVE_INCLUDES = 100 -def _include_cache_location(): - """Location to cache included configuration files.""" - return os.path.join(spack.paths.user_cache_path, "includes") - - class ConfigScope: - def __init__(self, name: str) -> None: + def __init__(self, name: str, included: bool = False) -> None: self.name = name self.writable = False self.sections = syaml.syaml_dict() self.prefer_modify = False + self.included = included - #: names of any included scopes + #: included configuration scopes self._included_scopes: Optional[List["ConfigScope"]] = None @property @@ -218,9 +216,15 @@ class DirectoryConfigScope(ConfigScope): """Config scope backed by a directory containing one file per section.""" def __init__( - self, name: str, path: str, *, writable: bool = True, prefer_modify: bool = True + self, + name: str, + path: str, + *, + writable: bool = True, + prefer_modify: bool = True, + included: bool = False, ) -> None: - super().__init__(name) + super().__init__(name, included) self.path = path self.writable = writable self.prefer_modify = prefer_modify @@ -263,8 +267,14 @@ def _write_section(self, section: str) -> None: try: filesystem.mkdirp(self.path) - with open(filename, "w", encoding="utf-8") as f: - syaml.dump_config(data, stream=f, default_flow_style=False) + fd, tmp = tempfile.mkstemp(dir=self.path, suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + syaml.dump_config(data, stream=f, default_flow_style=False) + filesystem.rename(tmp, filename) + except Exception: + os.unlink(tmp) + raise except (syaml.SpackYAMLError, OSError) as e: raise ConfigFileError(f"cannot write to '{filename}'") from e @@ -281,6 +291,7 @@ def __init__( yaml_path: Optional[List[str]] = None, writable: bool = True, prefer_modify: bool = True, + included: bool = False, ) -> None: """Similar to ``ConfigScope`` but can be embedded in another schema. @@ -299,7 +310,7 @@ def __init__( config: install_tree: $spack/opt/spack """ - super().__init__(name) + super().__init__(name, included) self._raw_data: Optional[YamlConfigDict] = None self.schema = schema self.path = path @@ -401,12 +412,14 @@ def _write_section(self, section: str) -> None: try: parent = os.path.dirname(self.path) filesystem.mkdirp(parent) - - tmp = os.path.join(parent, f".{os.path.basename(self.path)}.tmp") - with open(tmp, "w", encoding="utf-8") as f: - syaml.dump_config(data_to_write, stream=f, default_flow_style=False) - filesystem.rename(tmp, self.path) - + fd, tmp = tempfile.mkstemp(dir=parent, suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + syaml.dump_config(data_to_write, stream=f, default_flow_style=False) + filesystem.rename(tmp, self.path) + except Exception: + os.unlink(tmp) + raise except (syaml.SpackYAMLError, OSError) as e: raise ConfigFileError(f"cannot write to config file {str(e)}") from e @@ -538,7 +551,7 @@ def push_scope_incremental( # TODO: includes AND ensure properly sorted such that the order included # TODO: at the highest level is reflected in the value of an option that # TODO: is set in multiple included files. - # before pushing the scope itself, push any included scopes recursively, at same priority + # before pushing the scope itself, push included scopes recursively, at the same priority for included_scope in reversed(scope.included_scopes): if _depth + 1 > MAX_RECURSIVE_INCLUDES: # make sure we're not recursing endlessly mark = "" @@ -589,9 +602,9 @@ def remove_scope(self, scope_name: str) -> Optional[ConfigScope]: # transitively remove included scopes for included_scope in scope.included_scopes: - assert ( - included_scope.name in self.scopes - ), f"Included scope '{included_scope.name}' was never added to configuration!" + assert included_scope.name in self.scopes, ( + f"Included scope '{included_scope.name}' was never added to configuration!" + ) self.remove_scope(included_scope.name) return scope @@ -651,7 +664,8 @@ def _validate_scope(self, scope: Optional[str]) -> ConfigScope: else: raise ValueError( - f"Invalid config scope: '{scope}'. Must be one of {self.scopes.keys()}" + f"Invalid config scope: '{scope}'. Must be one of " + f"{[k for k in self.scopes.keys()]}" ) def get_config_filename(self, scope: str, section: str) -> str: @@ -753,24 +767,43 @@ def deepcopy_as_builtin( self.get_config(section, scope=scope), line_info=line_info ) - def _filter_overridden(self, scopes: List[ConfigScope]): + def _filter_overridden(self, scopes: List[ConfigScope], includes: bool = False): """Filter out overridden scopes. NOTE: this does not yet handle diamonds or nested `include::` in lists. It is sufficient for include::[] in an env, which allows isolation. + + The ``includes`` option controls whether to return all active scopes (``includes=False``) + or all scopes whose includes have not been overridden (``includes=True``). """ # find last override in scopes i = next((i for i, s in reversed(list(enumerate(scopes))) if s.override_include()), -1) if i < 0: return scopes # no overrides - keep = scopes[i].transitive_includes() + keep = _set(s.name for s in scopes[i:]) keep |= _set(s.name for s in self.scopes.priority_values(ConfigScopePriority.DEFAULTS)) - keep |= _set(s.name for s in scopes[i:]) + + if not includes: + # For all sections except for the include section: + # non-included scopes are still active, as are scopes included + # from the overriding scope + # Transitive scopes from the overriding scope are not included + keep |= _set([s.name for s in scopes[i].included_scopes]) + keep |= _set([s.name for s in scopes if not s.included]) # return scopes to keep, with order preserved return [s for s in scopes if s.name in keep] + @property + def active_include_section_scopes(self) -> List[ConfigScope]: + """Return a list of all scopes whose includes have not been overridden by include::. + + This is different from the active scopes because the ``spack`` scope can be active + while its includes are overwritten, as can the transitive includes from the overriding + scope.""" + return self._filter_overridden([s for s in self.scopes.values()], includes=True) + @property def active_scopes(self) -> List[ConfigScope]: """Return a list of scopes that have not been overridden by include::.""" @@ -804,8 +837,12 @@ def _get_config_memoized( merged_section: Dict[str, Any] = syaml.syaml_dict() updated_scopes = [] for config_scope in scopes: + if section == "include" and config_scope not in self.active_include_section_scopes: + continue + # read potentially cached data from the scope. data = config_scope.get_section(section) + if data and section == "include": # Include overrides are handled by `_filter_overridden` above. Any remaining # includes at this point are *not* actually overridden -- they're scopes with @@ -987,6 +1024,7 @@ class OptionalInclude: when: str optional: bool prefer_modify: bool + remote: bool _scopes: List[ConfigScope] def __init__(self, entry: dict): @@ -994,8 +1032,78 @@ def __init__(self, entry: dict): self.when = entry.get("when", "") self.optional = entry.get("optional", False) self.prefer_modify = entry.get("prefer_modify", False) + self.remote = False self._scopes = [] + @staticmethod + def _parent_scope_directory(parent_scope: Optional[ConfigScope]) -> Optional[str]: + """Return the directory of the parent scope, or ``None`` if unavailable. + + Normalizes ``SingleFileScope`` to its containing directory. + """ + path = getattr(parent_scope, "path", "") if parent_scope else "" + if not path: + return None + return os.path.dirname(path) if os.path.isfile(path) else path + + def base_directory( + self, path_or_url: str, parent_scope: Optional[ConfigScope] = None + ) -> Optional[str]: + """Return the local directory to use for this include. + + For remote includes this is the cache destination directory. + For local relative includes this is the working directory from which to resolve the path. + + Args: + path_or_url: path or URL of the include + parent_scope: including scope + + Returns: ``None`` for a local include without an enclosing parent scope; + an appropriate subdirectory of the enclosing (parent) scope's writable + directory (when available); otherwise a stable temporary directory. + """ + scope_dir = self._parent_scope_directory(parent_scope) + if not self.remote: + return scope_dir + + def _subdir(): + # Prefer the provided include name over the git repository name. + # If neither, use a hash of the url or path for uniqueness. + if self.name: + return self.name + + match = re.search(r"/([^/]+?)(\.git)?$", path_or_url) + if match: + if not os.path.splitext(match.group(1))[1]: + return match.group(1) + + return spack.util.hash.b32_hash(path_or_url)[-7:] + + # For remote includes, prefer a writable subdirectory of the parent scope. + if scope_dir and filesystem.can_write_to_dir(scope_dir): + assert parent_scope is not None + subdir = os.path.join("includes", _subdir()) + if parent_scope.name.startswith("env:"): + subdir = os.path.join(".spack-env", subdir) + return os.path.join(scope_dir, subdir) + + # Fall back to a stable, unique, temporary directory, logging the reason. + tmpdir = tempfile.gettempdir() + if path_or_url: + pre = self.name or getattr(parent_scope, "name", "") + subdir = f"{pre}:{path_or_url}" if pre else path_or_url + tmpdir = os.path.join(tmpdir, spack.util.hash.b32_hash(subdir)[-7:]) + + if not scope_dir: + tty.debug(f"No parent scope directory for include ({self}). Using {tmpdir}.") + else: + assert parent_scope is not None + tty.debug( + f"Parent scope {parent_scope.name}'s directory ({scope_dir}) is not writable. " + f"Using {tmpdir}." + ) + return tmpdir + def _scope( self, path: str, config_path: str, parent_scope: ConfigScope ) -> Optional[ConfigScope]: @@ -1011,72 +1119,98 @@ def _scope( Raises: ValueError: the required configuration path does not exist """ - assert self._valid_parent_scope( - parent_scope - ), "Optional includes must have valid parent_scope object" - - # use specified name if there is one - config_name = self.name - if not config_name: - # Try to use the relative path to create the included scope name - parent_path = getattr(parent_scope, "path", None) - if parent_path and str(parent_path) == os.path.commonprefix( - [parent_path, config_path] - ): - included_name = os.path.relpath(config_path, parent_path) - else: - included_name = config_path + # circular dependencies + import spack.util.path + + # Ignore included concrete environment files (i.e., ``spack.lock``) + # since they are not normal configuration (scope) files and their + # processing is handled when the environment is processed. + if path and os.path.basename(path) == "spack.lock": + tty.debug( + f"Ignoring inclusion of '{path}' since environment lock files " + "are processed elsewhere" + ) + return None + + # Ensure the parent scope is valid + self._validate_parent_scope(parent_scope) + + # Determine the configuration scope name + config_name = self.name or parent_scope.name + + # But ensure that name is unique if there are multiple paths. + if not self.name or len(getattr(self, "paths", [])) > 1: + parent_path = pathlib.Path(getattr(parent_scope, "path", "")) + real_path = pathlib.Path(spack.util.path.substitute_path_variables(path)) + + try: + included_name = real_path.relative_to(parent_path) + except ValueError: + included_name = real_path if sys.platform == "win32": # Clean windows path for use in config name that looks nicer # ie. The path: C:\\some\\path\\to\\a\\file # becomes C/some/path/to/a/file - included_name = included_name.replace("\\", "/") - included_name = included_name.replace(":", "") + included_name = included_name.as_posix().replace(":", "") - config_name = f"{parent_scope.name}:{included_name}" + config_name = f"{config_name}:{included_name}" - _, ext = os.path.splitext(config_path) - ext_is_yaml = ext == ".yaml" or ext == ".yml" - is_dir = os.path.isdir(config_path) - exists = os.path.exists(config_path) + # Type | Extension | RESULT + # -------- | --------- | --------- + # missing | none | Directory + # missing | yaml | File + # missing | other | No scope + # directory | none/any | Directory + # file | yaml | File + # file | other | Error + exists = os.path.exists(config_path) if not exists and not self.optional: - dest = f" at ({config_path})" if config_path != path else "" + dest = f" at ({config_path})" if config_path != os.path.normpath(path) else "" raise ValueError(f"Required path ({path}) does not exist{dest}") - if (exists and not is_dir) or ext_is_yaml: - # files are assumed to be SingleFileScopes + _, ext = os.path.splitext(config_path) + if os.path.isdir(config_path) or not ext: + # directories are treated as regular ConfigScopes + tty.debug(f"Creating DirectoryConfigScope {config_name} for '{config_path}'") + return DirectoryConfigScope( + config_name, config_path, prefer_modify=self.prefer_modify, included=True + ) + elif ext == ".yaml" or ext == ".yml": tty.debug(f"Creating SingleFileScope {config_name} for '{config_path}'") return SingleFileScope( config_name, config_path, spack.schema.merged.schema, prefer_modify=self.prefer_modify, + included=True, ) - - if ext and not is_dir: + elif exists: raise ValueError( - f"File-based scope does not exist yet: should have a .yaml/.yml extension \ -for file scopes, or no extension for directory scopes (currently {ext})" + f"Unsupported file-based scope: path ({path}) should have " + "a .yaml/.yml extension for file scopes, " + "or no extension for directory scopes" ) - # directories are treated as regular ConfigScopes - # assign by "default" - tty.debug(f"Creating DirectoryConfigScope {config_name} for '{config_path}'") - return DirectoryConfigScope(config_name, config_path, prefer_modify=self.prefer_modify) + # Nonexistent files without yaml extension are ignored + tty.debug(f"Ignoring missing config path ({path})") + return None - def _valid_parent_scope(self, parent_scope: ConfigScope) -> bool: + def _validate_parent_scope(self, parent_scope: ConfigScope): """Validates that a parent scope is a valid configuration object""" # enforced by type checking but those can always be # type: ignore'd - assert isinstance( - parent_scope, ConfigScope - ), f"Optional include must have valid parent scope,\ - of type ConfigScope; Type:{type(parent_scope)} is not valid." - # naive check that parent scope name isn't empty or just whitespace - return bool(re.sub(r"\s", "", parent_scope.name)) + assert isinstance(parent_scope, ConfigScope), ( + f"Includes must be within a configuration scope (ConfigScope), not {type(parent_scope)}" # noqa: E501 + ) + + assert parent_scope.name.strip(), "Parent scope of an include must have a name" def evaluate_condition(self) -> bool: + """Evaluate the include condition: + + Returns: ``True`` if the include condition is satisfied; else ``False``. + """ # circular dependencies import spack.spec @@ -1088,8 +1222,8 @@ def scopes(self, parent_scope: ConfigScope) -> List[ConfigScope]: Args: parent_scope: including scope - Returns: configuration scopes IF the when condition is satisfied; - otherwise, an empty list. + Returns: configuration scopes for configuration files IF the when + condition is satisfied; otherwise, an empty list. Raises: ValueError: the required configuration path does not exist @@ -1109,13 +1243,19 @@ class IncludePath(OptionalInclude): destination: Optional[str] def __init__(self, entry: dict): + # circular dependencies + import spack.util.path + super().__init__(entry) path_override_env_var = entry.get("path_override_env_var", "") if path_override_env_var and path_override_env_var in os.environ: - self.path = os.environ[path_override_env_var] + path = os.environ[path_override_env_var] else: - self.path = entry.get("path", "") + path = entry.get("path", "") + self.path = spack.util.path.substitute_path_variables(path) + self.sha256 = entry.get("sha256", "") + self.remote = "sha256" in entry self.destination = None def __repr__(self): @@ -1146,18 +1286,18 @@ def scopes(self, parent_scope: ConfigScope) -> List[ConfigScope]: tty.debug(f"Using existing scopes: {[s.name for s in self._scopes]}") return self._scopes - # Make sure to use the proper (default) working directory when obtaining - # the local path for a local file. - def work_dir(): - if not os.path.isabs(self.path) and hasattr(parent_scope, "path"): - if os.path.isfile(parent_scope.path): - return os.path.dirname(parent_scope.path) - if os.path.isdir(parent_scope.path): - return parent_scope.path - return os.getcwd() - - with filesystem.working_dir(work_dir()): - config_path = rfc_util.local_path(self.path, self.sha256, _include_cache_location) + # An absolute path does not need a local base directory. + if os.path.isabs(self.path): + tty.debug(f"The included path ({self}) is absolute so needs no base directory") + base = None + else: + base = self.base_directory(self.path, parent_scope) + + # Make sure to use a proper working directory when obtaining the local + # path for a local (or remote) file. + tty.debug(f"Local base directory for {self.path} is {base}") + + config_path = rfc_util.local_path(self.path, self.sha256, base) assert config_path self.destination = config_path @@ -1175,7 +1315,7 @@ def paths(self) -> List[str]: class GitIncludePaths(OptionalInclude): - repo: str + git: str branch: str commit: str tag: str @@ -1183,13 +1323,20 @@ class GitIncludePaths(OptionalInclude): destination: Optional[str] def __init__(self, entry: dict): + # circular dependencies + import spack.util.path + super().__init__(entry) - self.repo = entry.get("git", "") + self.git = spack.util.path.substitute_path_variables(entry.get("git", "")) + self.branch = entry.get("branch", "") self.commit = entry.get("commit", "") self.tag = entry.get("tag", "") - self._paths = entry.get("paths", []) + self._paths = [ + spack.util.path.substitute_path_variables(path) for path in entry.get("paths", []) + ] self.destination = None + self.remote = True if not self.branch and not self.commit and not self.tag: raise spack.error.ConfigError( @@ -1208,38 +1355,48 @@ def __repr__(self): identifier = f"commit={self.commit}, tag={self.tag}" return ( - f"GitIncludePaths({self.repo}, paths={self.paths}, " + f"GitIncludePaths('{self.name}', {self.git}, paths={self._paths}, " f"{identifier}, when='{self.when}', optional={self.optional})" ) - def _destination(self): - dir_name = spack.util.hash.b32_hash(self.repo)[-7:] - return os.path.join(_include_cache_location(), dir_name) + def _clone(self, parent_scope: ConfigScope) -> Optional[str]: + """Clone the repository. - def _clone(self) -> Optional[str]: - """Clone the repository.""" + Args: + parent_scope: enclosing scope + + Returns: destination path if cloned or ``None`` + """ if self.fetched(): - tty.debug(f"Repository ({self.repo}) already cloned to {self.destination}") + tty.debug(f"Repository ({self.git}) already cloned to {self.destination}") return self.destination - destination = self._destination() + # environment includes should be located under the environment + destination = self.base_directory(self.git, parent_scope) + assert destination, f"{self} requires a local cache directory" + tty.debug(f"Cloning {self.git} into {destination}") + with filesystem.working_dir(destination, create=True): if not os.path.exists(".git"): try: - spack.util.git.init_git_repo(self.repo) + tty.debug("Initializing the git repository") + spack.util.git.init_git_repo(self.git) except spack.util.executable.ProcessError as e: raise spack.error.ConfigError( - f"Unable to initialize repository ({self.repo}) under {destination}: {e}" + f"Unable to initialize repository ({self.git}) under {destination}: {e}" ) try: if self.commit: + tty.debug(f"Pulling commit {self.commit}") spack.util.git.pull_checkout_commit(self.commit) elif self.tag: + tty.debug(f"Pulling tag {self.tag}") spack.util.git.pull_checkout_tag(self.tag) elif self.branch: # if the branch already exists we should use the # previously configured remote + tty.debug(f"Pulling branch {self.branch}") try: git = spack.util.git.git(required=True) output = git("config", f"branch.{self.branch}.remote", output=str) @@ -1259,8 +1416,10 @@ def _clone(self) -> Optional[str]: self.destination = destination return self.destination - def fetched(self): - return self.destination is not None and os.path.join(self.destination, ".git") + def fetched(self) -> bool: + return bool(self.destination) and os.path.exists( + os.path.join(self.destination, ".git") # type: ignore[arg-type] + ) def scopes(self, parent_scope: ConfigScope) -> List[ConfigScope]: """Instantiate configuration scopes for the included paths. @@ -1284,14 +1443,14 @@ def scopes(self, parent_scope: ConfigScope) -> List[ConfigScope]: tty.debug(f"Using existing scopes: {[s.name for s in self._scopes]}") return self._scopes - destination = self._clone() - if destination is None: + destination = self._clone(parent_scope) + if not destination: raise spack.error.ConfigError(f"Unable to cache the include: {self}") scopes: List[ConfigScope] = [] - for relative_path in self.paths: - config_path = os.path.join(destination, relative_path) - scope = self._scope(relative_path, config_path, parent_scope) + for path in self.paths: + config_path = str(pathlib.Path(destination) / path) + scope = self._scope(path, config_path, parent_scope) if scope is not None: scopes.append(scope) @@ -1410,7 +1569,7 @@ def create() -> Configuration: #: This is the singleton configuration instance for Spack. -CONFIG: Configuration = lang.Singleton(create_incremental) # type: ignore +CONFIG = cast(Configuration, lang.Singleton(create_incremental)) def add_from_file(filename: str, scope: Optional[str] = None) -> None: @@ -1972,7 +2131,7 @@ def ensure_latest_format_fn(section: str) -> Callable[[YamlConfigDict], bool]: @contextlib.contextmanager def use_configuration( - *scopes_or_paths: Union[ScopeWithOptionalPriority, str] + *scopes_or_paths: Union[ScopeWithOptionalPriority, str], ) -> Generator[Configuration, None, None]: """Use the configuration scopes passed as arguments within the context manager. @@ -2110,7 +2269,7 @@ def __init__( super().__init__(message) def _get_mark(self, validation_error, data): - """Get the file/line mark fo a validation error from a Spack YAML file.""" + """Get the file/line mark for a validation error from a Spack YAML file.""" # Try various places, starting with instance and parent for obj in (validation_error.instance, validation_error.parent): diff --git a/lib/spack/spack/container/__init__.py b/lib/spack/spack/container/__init__.py index a8710f24ed4f45..16e414e624e2e1 100644 --- a/lib/spack/spack/container/__init__.py +++ b/lib/spack/spack/container/__init__.py @@ -4,6 +4,7 @@ """Package that provides functions and classes to generate container recipes from a Spack environment """ + import warnings import spack.vendor.jsonschema diff --git a/lib/spack/spack/container/images.py b/lib/spack/spack/container/images.py index 35d15efc28bf4b..222f689d461f55 100644 --- a/lib/spack/spack/container/images.py +++ b/lib/spack/spack/container/images.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Manages the details on the images used in the various stages.""" + import json import os import shlex diff --git a/lib/spack/spack/container/writers.py b/lib/spack/spack/container/writers.py index d78a48527eeefc..f08b2044a99140 100644 --- a/lib/spack/spack/container/writers.py +++ b/lib/spack/spack/container/writers.py @@ -4,6 +4,7 @@ """Writers for different kind of recipes and related convenience functions. """ + import copy import shlex from collections import namedtuple @@ -85,7 +86,7 @@ def _stage_base_images(images_config): # Check the OS is mentioned in the internal data stored in a JSON file images_json = data()["images"] if not any(os_name == operating_system for os_name in images_json): - msg = 'invalid operating system name "{0}". ' "[Allowed values are {1}]" + msg = 'invalid operating system name "{0}". [Allowed values are {1}]' msg = msg.format(operating_system, ", ".join(data()["images"])) raise ValueError(msg) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 1e87a3fee5ed2f..2b17da37d88dcd 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -16,6 +16,7 @@ provides a cache and a sanity checking mechanism for what is in the filesystem. """ + import contextlib import datetime import os @@ -24,6 +25,7 @@ import time from json import JSONDecoder from typing import ( + IO, Any, Callable, Container, @@ -687,7 +689,7 @@ def _write_to_file(self, stream): try: sjson.dump(database, stream) except (TypeError, ValueError) as e: - raise sjson.SpackJSONError("error writing JSON database:", str(e)) + raise sjson.SpackJSONError("error writing JSON database:", e) def _read_spec_from_dict(self, spec_reader, hash_key, installs, hash=ht.dag_hash): """Recursively construct a spec from a hash in a YAML database. @@ -751,6 +753,27 @@ def query_local_by_spec_hash(self, hash_key: str) -> Optional[InstallRecord]: with self.read_transaction(): return self._data.get(hash_key, None) + def _assign_build_spec( + self, + spec_reader: Type["spack.spec.SpecfileReaderBase"], + hash_key: str, + installs: dict, + data: Dict[str, InstallRecord], + ): + # Add dependencies from other records in the install DB to + # form a full spec. + spec = data[hash_key].spec + spec_node_dict = installs[hash_key]["spec"] + if "name" not in spec_node_dict: + # old format + spec_node_dict = spec_node_dict[spec.name] + if "build_spec" in spec_node_dict: + assert spec_reader.SPEC_VERSION >= 2, "SpecfileV1 spec cannot have build_spec" + _, bhash, _ = spec_reader.extract_build_spec_info_from_node_dict(spec_node_dict) + _, build_spec = self.query_by_spec_hash(bhash, data=data) + assert build_spec is not None, f"build_spec with hash {bhash} not found in database" + spec._build_spec = build_spec.spec + def _assign_dependencies( self, spec_reader: Type["spack.spec.SpecfileReaderBase"], @@ -792,24 +815,31 @@ def _assign_dependencies( def _read_from_file(self, filename: pathlib.Path, *, reindex: bool = False) -> None: """Fill database from file, do not maintain old data. + + Does not do any locking. + """ + with filename.open("r", encoding="utf-8") as f: + self._read_from_stream(f, reindex=reindex) + + def _read_from_stream(self, stream: IO[str], *, reindex: bool = False) -> None: + """Fill database from a text stream, do not maintain old data. Translate the spec portions from node-dict form to spec form. Does not do any locking. """ + source = getattr(stream, "name", None) or self._index_path try: # In the future we may use a stream of JSON objects, hence `raw_decode` for compat. - fdata, _ = JSONDecoder().raw_decode(filename.read_text(encoding="utf-8")) + fdata, _ = JSONDecoder().raw_decode(stream.read()) except Exception as e: - raise CorruptDatabaseError(f"error parsing database at {filename}:", str(e)) from e + raise CorruptDatabaseError(f"error parsing database at {source}:", str(e)) from e if fdata is None: return def check(cond, msg): if not cond: - raise CorruptDatabaseError( - f"Spack database is corrupt: {msg}", str(self._index_path) - ) + raise CorruptDatabaseError(f"Spack database is corrupt: {msg}", str(source)) check("database" in fdata, "no 'database' attribute in JSON DB.") @@ -831,7 +861,7 @@ def invalid_record(hash_key, error): return CorruptDatabaseError( f"Invalid record in Spack database: hash: {hash_key}, cause: " f"{type(error).__name__}: {error}", - str(self._index_path), + str(source), ) # Build up the database in three passes: @@ -865,6 +895,7 @@ def invalid_record(hash_key, error): # Pass 2: Assign dependencies once all specs are created. for hash_key in data: try: + self._assign_build_spec(spec_reader, hash_key, installs, data) self._assign_dependencies(spec_reader, hash_key, installs, data) except MissingDependenciesError: raise @@ -940,8 +971,10 @@ def reindex(self): # ignore errors if we need to rebuild a corrupt database. def _read_suppress_error(): try: - if self._index_path.is_file(): - self._read_from_file(self._index_path, reindex=True) + with self._index_path.open("r", encoding="utf-8") as f: + self._read_from_stream(f, reindex=True) + except FileNotFoundError: + pass except (CorruptDatabaseError, DatabaseNotReadableError): self._data = {} self._installed_prefixes = set() @@ -1125,24 +1158,28 @@ def _write(self, type=None, value=None, traceback=None): def _read(self): """Re-read Database from the data in the set location. This does no locking.""" - if self._index_path.is_file(): + try: + index_file = self._index_path.open("r", encoding="utf-8") + except FileNotFoundError: + if self.is_upstream: + tty.warn(f"upstream not found: {self._index_path}") + return + + with index_file as f: current_verifier = "" if _use_uuid: try: - with self._verifier_path.open("r", encoding="utf-8") as f: - current_verifier = f.read() + with self._verifier_path.open("r", encoding="utf-8") as vf: + current_verifier = vf.read() except BaseException: pass if (current_verifier != self.last_seen_verifier) or (current_verifier == ""): self.last_seen_verifier = current_verifier # Read from file if a database exists - self._read_from_file(self._index_path) + self._read_from_stream(f) elif self._state_is_inconsistent: - self._read_from_file(self._index_path) + self._read_from_stream(f) self._state_is_inconsistent = False - return - elif self.is_upstream: - tty.warn(f"upstream not found: {self._index_path}") def _add( self, @@ -1191,6 +1228,9 @@ def _add( allow_missing=allow_missing or edge.depflag & (dt.BUILD | dt.TEST) == edge.depflag, ) + if spec.spliced: + self._add(spec.build_spec, explicit=False, allow_missing=True) + # Make sure the directory layout agrees whether the spec is installed if not spec.external and self.layout: path = self.layout.path_for_spec(spec) diff --git a/lib/spack/spack/dependency.py b/lib/spack/spack/dependency.py index 5415ddf4ae7424..66642c5dc8a30c 100644 --- a/lib/spack/spack/dependency.py +++ b/lib/spack/spack/dependency.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Data structures that represent Spack's dependency relationships.""" + from typing import TYPE_CHECKING, Dict, List, Type import spack.deptypes as dt diff --git a/lib/spack/spack/deptypes.py b/lib/spack/spack/deptypes.py index 0ebaaa2a21cbfe..470d8c4b27ca70 100644 --- a/lib/spack/spack/deptypes.py +++ b/lib/spack/spack/deptypes.py @@ -148,7 +148,7 @@ def flag_to_chars(depflag: DepFlag) -> str: For a single dependency, this just indicates that the dependency has the indicated deptypes. For a list of dependnecies, this shows - whether ANY dpeendency in the list has the deptypes (so the deptypes + whether ANY dependency in the list has the deptypes (so the deptypes are merged).""" return "".join( t_str[0] if t_flag & depflag else " " for t_str, t_flag in zip(ALL_TYPES, ALL_FLAGS) diff --git a/lib/spack/spack/detection/common.py b/lib/spack/spack/detection/common.py index 866cc951566fa1..ecf1db9d9a62bb 100644 --- a/lib/spack/spack/detection/common.py +++ b/lib/spack/spack/detection/common.py @@ -12,6 +12,7 @@ The module also contains other functions that might be useful across different detection mechanisms. """ + import glob import itertools import os @@ -185,7 +186,7 @@ def library_prefix(library_dir: str) -> str: assert os.path.isdir(library_dir) components = library_dir.split(os.sep) - # covert to lowercase to match lib, LIB, Lib, etc. + # convert to lowercase to match lib, LIB, Lib, etc. lowered_components = library_dir.lower().split(os.sep) if "lib64" in lowered_components: idx = lowered_components.index("lib64") diff --git a/lib/spack/spack/detection/path.py b/lib/spack/spack/detection/path.py index 183cb3d1fbba57..292d36468b5253 100644 --- a/lib/spack/spack/detection/path.py +++ b/lib/spack/spack/detection/path.py @@ -4,6 +4,7 @@ """Detection of software installed in the system, based on paths inspections and running executables. """ + import collections import concurrent.futures import os @@ -268,6 +269,7 @@ def detect_specs( return [] result = [] + resolved_specs: Dict[spack.spec.Spec, str] = {} # spec -> prefix of first detection for candidate_path, items_in_prefix in _group_by_prefix( spack.llnl.util.lang.dedupe(paths) ).items(): @@ -297,21 +299,19 @@ def detect_specs( f"part of the package {pkg.name}: {files}" ) - resolved_specs: Dict[spack.spec.Spec, str] = {} # spec -> exe found for the spec for spec in specs: prefix = self.prefix_from_path(path=candidate_path) if not prefix: continue if spec in resolved_specs: - prior_prefix = ", ".join(_convert_to_iterable(resolved_specs[spec])) - spack.llnl.util.tty.debug( - f"Files in {candidate_path} and {prior_prefix} are both associated" - f" with the same spec {str(spec)}" + prior_prefix = resolved_specs[spec] + warnings.warn( + f'"{spec}" detected in "{prefix}" was already detected in "{prior_prefix}"' ) continue - resolved_specs[spec] = candidate_path + resolved_specs[spec] = prefix try: # Validate the spec calling a package specific method pkg_cls = repo_path.get_pkg_class(spec.name) @@ -441,7 +441,7 @@ def by_path( if max_workers == 1: executor = spack.util.parallel.SequentialExecutor() else: - executor = spack.util.parallel.make_concurrent_executor(max_workers, require_fork=False) + executor = spack.util.parallel.make_concurrent_executor(max_workers) with executor: for pkg in packages_to_search: executable_future = executor.submit( diff --git a/lib/spack/spack/detection/test.py b/lib/spack/spack/detection/test.py index 3741c1a18c680a..2c66cc8884d347 100644 --- a/lib/spack/spack/detection/test.py +++ b/lib/spack/spack/detection/test.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Create and run mock e2e tests for package detection.""" + import collections import contextlib import pathlib diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 26bdb8d3cad79a..f9cdb42e2810d3 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -39,6 +39,7 @@ def example_directive(arg1, arg2): def _execute_example_directive(pkg, arg1, arg2): # modify pkg.example based on arg1 and arg2 """ + import collections import collections.abc import os @@ -57,15 +58,10 @@ def _execute_example_directive(pkg, arg1, arg2): import spack.util.crypto import spack.variant from spack.dependency import Dependency -from spack.directives_meta import DirectiveError, DirectiveMeta +from spack.directives_meta import DirectiveError, directive, get_spec from spack.resource import Resource -from spack.version import ( - GitVersion, - Version, - VersionChecksumError, - VersionError, - VersionLookupError, -) +from spack.spec import EMPTY_SPEC +from spack.version import StandardVersion, VersionChecksumError, VersionError __all__ = [ "DirectiveError", @@ -97,7 +93,7 @@ def _execute_example_directive(pkg, arg1, arg2): PatchesType = Union[Patcher, str, List[Union[Patcher, str]]] -def _make_when_spec(value: WhenType) -> Optional[spack.spec.Spec]: +def _make_when_spec(value: Union[WhenType, Tuple[str, ...]]) -> Optional[spack.spec.Spec]: """Create a ``Spec`` that indicates when a directive should be applied. Directives with ``when`` specs, e.g.: @@ -121,12 +117,25 @@ def _make_when_spec(value: WhenType) -> Optional[spack.spec.Spec]: Arguments: value: a conditional Spec, constant ``bool``, or None if not supplied - value indicating when a directive should be applied. + value indicating when a directive should be applied. It can also be a tuple of when + conditions (as strings) to be combined together. """ + # This branch is never taken, but our WhenType type annotation allows it, so handle it too. if isinstance(value, spack.spec.Spec): return value + if isinstance(value, tuple): + assert value, "when stack cannot be empty" + # avoid a copy when there's only one condition + if len(value) == 1: + return get_spec(value[0]) + # reduce the when-stack to a single spec by combining all constraints. + combined_spec = spack.spec.Spec(value[0]) + for cond in value[1:]: + combined_spec._constrain_symbolically(get_spec(cond)) + return combined_spec + # Unsatisfiable conditions are discarded by the caller, and never # added to the package class if value is False: @@ -136,17 +145,16 @@ def _make_when_spec(value: WhenType) -> Optional[spack.spec.Spec]: # represent this by returning the unconstrained `Spec()`, which is # always satisfied. if value is None or value is True: - return spack.spec.Spec() + return EMPTY_SPEC # This is conditional on the spec - return spack.spec.Spec(value) + return get_spec(value) SubmoduleCallback = Callable[[spack.package_base.PackageBase], Union[str, List[str], bool]] -directive = DirectiveMeta.directive -@directive("versions") +@directive("versions", supports_when=False) def version( ver: Union[str, int], # this positional argument is deprecated, use sha256=... instead @@ -240,7 +248,7 @@ def _execute_version(pkg: PackageType, ver: Union[str, int], kwargs: dict): and not pkg.has_code ): raise VersionChecksumError( - f"{pkg.name}: Checksums not allowed in no-code packages " f"(see '{ver}' version)." + f"{pkg.name}: Checksums not allowed in no-code packages (see '{ver}' version)." ) if not isinstance(ver, (int, str)): @@ -248,15 +256,7 @@ def _execute_version(pkg: PackageType, ver: Union[str, int], kwargs: dict): f"{pkg.name}: declared version '{ver!r}' in package should be a string or int." ) - # Declared versions are concrete - version = Version(ver) - - if isinstance(version, GitVersion) and not hasattr(pkg, "git") and "git" not in kwargs: - args = ", ".join(f"{argname}='{value}'" for argname, value in kwargs.items()) - raise VersionLookupError( - f"{pkg.name}: spack version directives cannot include git hashes fetched from URLs.\n" - f" version('{ver}', {args})" - ) + version = StandardVersion.from_string(str(ver)) # Store kwargs for the package to later with a fetch_strategy. pkg.versions[version] = kwargs @@ -292,14 +292,15 @@ def _execute_conflicts(pkg: PackageType, conflict_spec, when, msg): # Save in a list the conflicts and the associated custom messages conflict_spec_list = pkg.conflicts.setdefault(when_spec, []) msg_with_name = f"{pkg.name}: {msg}" if msg is not None else msg - conflict_spec_list.append((spack.spec.Spec(conflict_spec), msg_with_name)) + conflict_spec_list.append((get_spec(conflict_spec), msg_with_name)) -@directive("dependencies") +@directive("dependencies", can_patch_dependencies=True) def depends_on( spec: SpecType, when: WhenType = None, type: DepType = dt.DEFAULT_TYPES, + *, patches: Optional[PatchesType] = None, ): """Declare a dependency on another package. @@ -316,18 +317,18 @@ def depends_on( patches: single result of :py:func:`patch` directive, a ``str`` to be passed to ``patch``, or a list of these """ - dep_spec = spack.spec.Spec(spec) - return partial(_execute_depends_on, spec=dep_spec, when=when, type=type, patches=patches) + return partial(_execute_depends_on, spec=spec, when=when, type=type, patches=patches) def _execute_depends_on( pkg: PackageType, - spec: spack.spec.Spec, + spec: Union[str, spack.spec.Spec], *, when: WhenType = None, type: DepType = dt.DEFAULT_TYPES, patches: Optional[PatchesType] = None, ): + spec = get_spec(spec) if isinstance(spec, str) else spec when_spec = _make_when_spec(when) if not when_spec: return @@ -362,10 +363,6 @@ def _execute_depends_on( elif not isinstance(patches, (list, tuple)): patches = [patches] - # auto-call patch() directive on any strings in patch list - patches = [patch(p) if isinstance(p, str) else p for p in patches] - assert all(callable(p) for p in patches) - # this is where we actually add the dependency to this package deps_by_name = pkg.dependencies.setdefault(when_spec, {}) dependency = deps_by_name.get(spec.name) @@ -388,8 +385,12 @@ def _execute_depends_on( dependency.depflag |= depflag # apply patches to the dependency - for execute_patch in patches: - execute_patch(dependency) + for patch in patches: + if isinstance(patch, str): + _execute_patch(dependency, url_or_filename=patch) + else: + assert callable(patch), f"Invalid patch argument: {patch!r}" + patch(dependency) @directive("disable_redistribute") @@ -411,8 +412,7 @@ def _execute_redistribute( return elif (source is True) or (binary is True): raise DirectiveError( - "Source/binary distribution are true by default, they can only " - "be explicitly disabled." + "Source/binary distribution are true by default, they can only be explicitly disabled." ) if source is None: @@ -424,7 +424,7 @@ def _execute_redistribute( if not when_spec: return if source is False: - max_constraint = spack.spec.Spec(f"{pkg.name}@{when_spec.versions}") + max_constraint = get_spec(f"{pkg.name}@{when_spec.versions}") if not max_constraint.satisfies(when_spec): raise DirectiveError("Source distribution can only be disabled for versions") @@ -440,11 +440,12 @@ def _execute_redistribute( ) -@directive(("extendees", "dependencies")) +@directive(("extendees", "dependencies"), can_patch_dependencies=True) def extends( spec: str, when: WhenType = None, type: DepType = ("build", "run"), + *, patches: Optional[PatchesType] = None, ): """Same as :func:`depends_on`, but also adds this package to the extendee list. @@ -465,14 +466,14 @@ def _execute_extends( if not when_spec: return - dep_spec = spack.spec.Spec(spec) + dep_spec = get_spec(spec) _execute_depends_on(pkg, dep_spec, when=when, type=type, patches=patches) # When extending python, also add a dependency on python-venv. This is done so that # Spack environment views are Python virtual environments. if dep_spec.name == "python" and not pkg.name == "python-venv": - _execute_depends_on(pkg, spack.spec.Spec("python-venv"), when=when, type=("build", "run")) + _execute_depends_on(pkg, "python-venv", when=when, type=("build", "run")) pkg.extendees[dep_spec.name] = (dep_spec, when_spec) @@ -497,21 +498,16 @@ def _execute_provides(pkg: PackageType, specs: Tuple[SpecType, ...], when: WhenT if not when_spec: return - # ``when`` specs for ``provides()`` need a name, as they are used - # to build the ProviderIndex. - when_spec.name = pkg.name - - spec_objs = [spack.spec.Spec(x) for x in specs] + spec_objs = [get_spec(x) for x in specs] spec_names = [x.name for x in spec_objs] if len(spec_names) > 1: pkg.provided_together.setdefault(when_spec, []).append(set(spec_names)) for provided_spec in spec_objs: if pkg.name == provided_spec.name: - raise CircularReferenceError("Package '%s' cannot provide itself." % pkg.name) + raise CircularReferenceError(f"Package '{pkg.name}' cannot provide itself.") - provided_set = pkg.provided.setdefault(when_spec, set()) - provided_set.add(provided_spec) + pkg.provided.setdefault(when_spec, set()).add(provided_spec) @directive("splice_specs") @@ -549,7 +545,7 @@ def _execute_can_splice( ) if when_spec is None: return - pkg.splice_specs[when_spec] = (spack.spec.Spec(target), match_variants) + pkg.splice_specs[when_spec] = (get_spec(target), match_variants) @directive("patches") @@ -596,12 +592,12 @@ def patch( def _execute_patch( pkg_or_dep: Union[PackageType, Dependency], url_or_filename: str, - level: int, - when: WhenType, - working_dir: str, - reverse: bool, - sha256: Optional[str], - archive_sha256: Optional[str], + level: int = 1, + when: WhenType = None, + working_dir: str = ".", + reverse: bool = False, + sha256: Optional[str] = None, + archive_sha256: Optional[str] = None, ) -> None: pkg = pkg_or_dep.pkg if isinstance(pkg_or_dep, Dependency) else pkg_or_dep @@ -906,7 +902,7 @@ def maintainers(*names: str): def _execute_maintainer(pkg: PackageType, names: Tuple[str, ...]): - maintainers = set(getattr(pkg, "maintainers", [])) + maintainers = set(pkg.maintainers) maintainers.update(names) pkg.maintainers = sorted(maintainers) @@ -939,10 +935,10 @@ def _execute_license(pkg: PackageType, license_identifier: str, when: Optional[U for other_when_spec in pkg.licenses: if when_spec.intersects(other_when_spec): when_message = "" - if when_spec != _make_when_spec(None): + if when_spec != EMPTY_SPEC: when_message = f"when {when_spec}" other_when_message = "" - if other_when_spec != _make_when_spec(None): + if other_when_spec != EMPTY_SPEC: other_when_message = f"when {other_when_spec}" err_msg = ( f"{pkg.name} is specified as being licensed as {license_identifier} " @@ -1007,7 +1003,7 @@ def _execute_requires( # Save in a list the requirements and the associated custom messages requirement_list = pkg.requirements.setdefault(when_spec, []) msg_with_name = f"{pkg.name}: {msg}" if msg is not None else msg - requirements = tuple(spack.spec.Spec(s) for s in requirement_specs) + requirements = tuple(get_spec(s) for s in requirement_specs) requirement_list.append((requirements, policy, msg_with_name)) diff --git a/lib/spack/spack/directives_meta.py b/lib/spack/spack/directives_meta.py index 174ef29ed4500f..fb5ed49a726e7f 100644 --- a/lib/spack/spack/directives_meta.py +++ b/lib/spack/spack/directives_meta.py @@ -2,233 +2,301 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import collections.abc +import collections import functools -from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Type, Union +from typing import Any, Callable, Dict, List, Set, Tuple, Type, TypeVar, Union + +from spack.vendor.typing_extensions import ParamSpec import spack.error -import spack.llnl.util.lang import spack.repo import spack.spec +from spack.llnl.util.lang import dedupe + +P = ParamSpec("P") +R = TypeVar("R") #: Names of possible directives. This list is mostly populated using the @directive decorator. #: Some directives leverage others and in that case are not automatically added. directive_names = ["build_system"] +SPEC_CACHE: Dict[str, spack.spec.Spec] = {} + + +def get_spec(spec_str: str) -> spack.spec.Spec: + """Get a spec from the cache, or create it if not present.""" + if spec_str not in SPEC_CACHE: + SPEC_CACHE[spec_str] = spack.spec._ImmutableSpec(spec_str) + return SPEC_CACHE[spec_str] + class DirectiveMeta(type): """Flushes the directives that were temporarily stored in the staging area into the package. """ - # Set of all known directives + #: Registry of {directive_name: [list_of_dicts_it_modifies]} populated by @directive + _directive_to_dicts: Dict[str, Tuple[str, ...]] = {} + #: Inverted index of {dict_name: [list_of_directives_modifying_it]} + _dict_to_directives: Dict[str, List[str]] = collections.defaultdict(list) + #: Maps dictionary name to its descriptor instance + _descriptor_cache: Dict[str, "DirectiveDictDescriptor"] = {} + #: Set of all known directive dictionary names from `@directive(dicts=...)` _directive_dict_names: Set[str] = set() - _directives_to_be_executed: List[Callable] = [] - _when_constraints_from_context: List[spack.spec.Spec] = [] - _default_args: List[dict] = [] + #: Lists of directives to be executed for the class being defined, grouped by directive + #: function name (e.g. "depends_on", "version", etc.) + _directives_to_be_executed: Dict[str, List[Callable]] = collections.defaultdict(list) + #: Stack of when constraints from `with when(...)` context managers + _when_constraints_stack: List[str] = [] + #: Stack of default args from `with default_args(...)` context managers + _default_args_stack: List[dict] = [] + #: This property is set *automatically* during class definition as directives are invoked, + #: if any ``depends_on`` or ``extends`` calls include patches for dependencies. This flag can + #: be used as an optimization to detect whether a package provides patches for dependencies, + #: without triggering the expensive deferred execution of those directives (without populating + #: the ``dependencies`` dictionary). + _patches_dependencies: bool = False def __new__( cls: Type["DirectiveMeta"], name: str, bases: tuple, attr_dict: dict ) -> "DirectiveMeta": - # Initialize the attribute containing the list of directives - # to be executed. Here we go reversed because we want to execute - # commands: - # 1. in the order they were defined - # 2. following the MRO - attr_dict["_directives_to_be_executed"] = [] - for base in reversed(bases): - try: - directive_from_base = base._directives_to_be_executed - attr_dict["_directives_to_be_executed"].extend(directive_from_base) - except AttributeError: - # The base class didn't have the required attribute. - # Continue searching - pass - - # De-duplicates directives from base classes - attr_dict["_directives_to_be_executed"] = [ - x for x in spack.llnl.util.lang.dedupe(attr_dict["_directives_to_be_executed"]) - ] - - # Move things to be executed from module scope (where they - # are collected first) to class scope - if DirectiveMeta._directives_to_be_executed: - attr_dict["_directives_to_be_executed"].extend( - DirectiveMeta._directives_to_be_executed - ) - DirectiveMeta._directives_to_be_executed = [] + attr_dict["_patches_dependencies"] = DirectiveMeta._patches_dependencies + # Initialize the attribute containing the list of directives to be executed. Here we go + # reversed because we want to execute commands in the order they were defined, following + # the MRO. + merged: Dict[str, List[Callable]] = {} + sources = [getattr(b, "_directives_to_be_executed", None) or {} for b in reversed(bases)] + for source in sources: + for key, directive_list in source.items(): + merged.setdefault(key, []).extend(directive_list) + + merged = {key: list(dedupe(directive_list)) for key, directive_list in merged.items()} + + # Add current class's directives (no deduplication needed here) + for key, directive_list in DirectiveMeta._directives_to_be_executed.items(): + merged.setdefault(key, []).extend(directive_list) + + attr_dict["_directives_to_be_executed"] = merged + + DirectiveMeta._directives_to_be_executed.clear() + DirectiveMeta._patches_dependencies = False + + # Add descriptors for all known directive dictionaries + for dict_name in DirectiveMeta._directive_dict_names: + # Where the actual data will be stored + attr_dict[f"_{dict_name}"] = None + # Descriptor to lazily initialize and populate the dictionary + attr_dict[dict_name] = DirectiveMeta._get_descriptor(dict_name) return super(DirectiveMeta, cls).__new__(cls, name, bases, attr_dict) def __init__(cls: "DirectiveMeta", name: str, bases: tuple, attr_dict: dict): - # The instance is being initialized: if it is a package we must ensure - # that the directives are called to set it up. - if spack.repo.is_package_module(cls.__module__): - # Ensure the presence of the dictionaries associated with the directives. - # All dictionaries are defaultdicts that create lists for missing keys. - for d in DirectiveMeta._directive_dict_names: - setattr(cls, d, {}) - - # Lazily execute directives - for directive in cls._directives_to_be_executed: + # Historically, maintainers was not a directive. They were simply set as class + # attributes `maintainers = ["alice", "bob"]`. Therefore, we execute these directives + # eagerly. + for directive in cls._directives_to_be_executed.get("maintainers", ()): directive(cls) + super(DirectiveMeta, cls).__init__(name, bases, attr_dict) - # Ignore any directives executed *within* top-level - # directives by clearing out the queue they're appended to - DirectiveMeta._directives_to_be_executed = [] + @staticmethod + def register_directive(name: str, dicts: Tuple[str, ...]) -> None: + """Called by @directive to register relationships.""" + DirectiveMeta._directive_to_dicts[name] = dicts + for d in dicts: + DirectiveMeta._dict_to_directives[d].append(name) - super(DirectiveMeta, cls).__init__(name, bases, attr_dict) + @staticmethod + def _get_descriptor(name: str) -> "DirectiveDictDescriptor": + """Returns a singleton descriptor for the given dictionary name.""" + if name not in DirectiveMeta._descriptor_cache: + DirectiveMeta._descriptor_cache[name] = DirectiveDictDescriptor(name) + return DirectiveMeta._descriptor_cache[name] @staticmethod - def push_to_context(when_spec: spack.spec.Spec) -> None: + def push_when_constraint(when_spec: str) -> None: """Add a spec to the context constraints.""" - DirectiveMeta._when_constraints_from_context.append(when_spec) + DirectiveMeta._when_constraints_stack.append(when_spec) @staticmethod - def pop_from_context() -> spack.spec.Spec: + def pop_when_constraint() -> str: """Pop the last constraint from the context""" - return DirectiveMeta._when_constraints_from_context.pop() + return DirectiveMeta._when_constraints_stack.pop() @staticmethod def push_default_args(default_args: Dict[str, Any]) -> None: """Push default arguments""" - DirectiveMeta._default_args.append(default_args) + DirectiveMeta._default_args_stack.append(default_args) @staticmethod def pop_default_args() -> dict: """Pop default arguments""" - return DirectiveMeta._default_args.pop() + return DirectiveMeta._default_args_stack.pop() @staticmethod - def _remove_directives(arg): - # If any of the arguments are executors returned by a directive passed as an argument, - # don't execute them lazily. Instead, let the called directive handle them. This allows - # nested directive calls in packages. The caller can return the directive if it should be - # queued. Nasty, but it's the best way I can think of to avoid side effects if directive - # results are passed as args - directives = DirectiveMeta._directives_to_be_executed - if isinstance(arg, (list, tuple)): - # Descend into args that are lists or tuples - for a in arg: - DirectiveMeta._remove_directives(a) - else: + def _remove_kwarg_value_directives_from_queue(value) -> None: + """Remove directives found in a kwarg value from the execution queue.""" + # Certain keyword argument values of directives may themselves be (lists of) directives. An + # example of this is ``depends_on(..., patches=[patch(...), ...])``. In that case, we + # should not execute those directives as part of the current package, but let the called + # directive handle them. This function removes such directives from the execution queue. + if isinstance(value, (list, tuple)): + for item in value: + DirectiveMeta._remove_kwarg_value_directives_from_queue(item) + elif callable(value): # directives are always callable # Remove directives args from the exec queue - remove = next((d for d in directives if d is arg), None) - if remove is not None: - directives.remove(remove) + for lst in DirectiveMeta._directives_to_be_executed.values(): + for directive in lst: + if value is directive: + lst.remove(directive) # iterations ends, so mutation is fine + break @staticmethod - def directive(dicts: Optional[Union[Sequence[str], str]] = None) -> Callable: - """Decorator for Spack directives. + def _get_execution_plan(target_dict: str) -> Tuple[List[str], List[str]]: + """Calculates the closure of dicts and directives needed to populate target_dict.""" + dicts_involved = {target_dict} + directives_involved = set() + stack = [target_dict] + + while stack: + current_dict = stack.pop() + + for directive_name in DirectiveMeta._dict_to_directives.get(current_dict, ()): + if directive_name in directives_involved: + continue + + directives_involved.add(directive_name) + + for other_dict in DirectiveMeta._directive_to_dicts[directive_name]: + if other_dict not in dicts_involved: + dicts_involved.add(other_dict) + stack.append(other_dict) + + return sorted(dicts_involved), sorted(directives_involved) - Spack directives allow you to modify a package while it is being - defined, e.g. to add version or dependency information. Directives - are one of the key pieces of Spack's package "language", which is - embedded in python. - Here's an example directive: +class DirectiveDictDescriptor: + """A descriptor that lazily executes directives on first access.""" - .. code-block:: python + def __init__(self, name: str): + self.name = name + self.private_name = f"_{name}" + self.dicts_to_init, self.directives_to_run = DirectiveMeta._get_execution_plan(name) + + def __get__(self, obj, objtype=None): + val = getattr(objtype, self.private_name) + if val is not None: + return val + + # The None value is a sentinel for "not yet initialized". + for dictionary in self.dicts_to_init: + if getattr(objtype, f"_{dictionary}") is None: + setattr(objtype, f"_{dictionary}", {}) + + # Populate these dictionaries by running all directives that modify them + for directive_name in self.directives_to_run: + directives = objtype._directives_to_be_executed.get(directive_name) + if directives: + for directive in directives: + directive(objtype) + + return getattr(objtype, self.private_name) + + +class directive: + def __init__( + self, + dicts: Union[Tuple[str, ...], str] = (), + supports_when: bool = True, + can_patch_dependencies: bool = False, + ) -> None: + """Decorator for Spack directives. + + Spack directives allow you to modify a package while it is being defined, e.g. to add + version or dependency information. Directives are one of the key pieces of Spack's + package "language", which is embedded in python. + + Here's an example directive:: @directive(dicts="versions") def version(pkg, ...): ... - This directive allows you write: - - .. code-block:: python + This directive allows you write:: class Foo(Package): version(...) The ``@directive`` decorator handles a couple things for you: - 1. Adds the class scope (pkg) as an initial parameter when - called, like a class method would. This allows you to modify - a package from within a directive, while the package is still - being defined. - - 2. It automatically adds a dictionary called ``versions`` to the - package so that you can refer to pkg.versions. - - The ``(dicts="versions")`` part ensures that ALL packages in Spack - will have a ``versions`` attribute after they're constructed, and - that if no directive actually modified it, it will just be an - empty dict. - - This is just a modular way to add storage attributes to the - Package class, and it's how Spack gets information from the - packages to the core. + 1. Adds the class scope (pkg) as an initial parameter when called, like a class method + would. This allows you to modify a package from within a directive, while the package is + still being defined. + + 2. It automatically adds a dictionary called ``versions`` to the package so that you can + refer to pkg.versions. + + Arguments: + dicts: A tuple of names of dictionaries to add to the package class if they don't + already exist. + supports_when: If True, the directive can be used within a ``with when(...)`` context + manager. (To be removed when all directives support ``when=`` arguments.) + can_patch_dependencies: If True, the directive can patch dependencies. This is used to + identify nested directives so they can be removed from the execution queue, and to + mark the package as patching dependencies. """ if isinstance(dicts, str): dicts = (dicts,) - if not isinstance(dicts, collections.abc.Sequence): - message = "dicts arg must be list, tuple, or string. Found {0}" - raise TypeError(message.format(type(dicts))) - # Add the dictionary names if not already there - DirectiveMeta._directive_dict_names |= set(dicts) - - # This decorator just returns the directive functions - def _decorator(decorated_function: Callable) -> Callable: - directive_names.append(decorated_function.__name__) - - @functools.wraps(decorated_function) - def _wrapper(*args, **_kwargs): - # First merge default args with kwargs - kwargs = dict() - for default_args in DirectiveMeta._default_args: + DirectiveMeta._directive_dict_names.update(dicts) + + self.supports_when = supports_when + self.can_patch_dependencies = can_patch_dependencies + self.dicts = tuple(dicts) + + def __call__(self, decorated_function: Callable[P, R]) -> Callable[P, R]: + directive_names.append(decorated_function.__name__) + DirectiveMeta.register_directive(decorated_function.__name__, self.dicts) + + @functools.wraps(decorated_function) + def _wrapper(*args, **_kwargs): + # First merge default args with kwargs + if DirectiveMeta._default_args_stack: + kwargs = {} + for default_args in DirectiveMeta._default_args_stack: kwargs.update(default_args) kwargs.update(_kwargs) - - # Inject when arguments from the context - if DirectiveMeta._when_constraints_from_context: - # Check that directives not yet supporting the when= argument - # are not used inside the context manager - if decorated_function.__name__ == "version": - msg = ( - 'directive "{0}" cannot be used within a "when"' - ' context since it does not support a "when=" ' - "argument" - ) - msg = msg.format(decorated_function.__name__) - raise DirectiveError(msg) - - when_constraints = [ - spack.spec.Spec(x) for x in DirectiveMeta._when_constraints_from_context - ] - if kwargs.get("when"): - when_constraints.append(spack.spec.Spec(kwargs["when"])) - - when_spec = spack.spec.Spec() - for current in when_constraints: - when_spec._constrain_symbolically(current, deps=True) - kwargs["when"] = when_spec - - DirectiveMeta._remove_directives(args) - DirectiveMeta._remove_directives(list(kwargs.values())) - - # A directive returns either something that is callable on a - # package or a sequence of them - result = decorated_function(*args, **kwargs) - - # ...so if it is not a sequence make it so - values = result - if not isinstance(values, collections.abc.Sequence): - values = (values,) - - DirectiveMeta._directives_to_be_executed.extend(values) - - # wrapped function returns same result as original so - # that we can nest directives - return result - - return _wrapper - - return _decorator + else: + kwargs = _kwargs + + # Inject when arguments from the `with when(...)` stack. + if DirectiveMeta._when_constraints_stack: + if not self.supports_when: + raise DirectiveError( + f'directive "{decorated_function.__name__}" cannot be used within a ' + '"when" context since it does not support a "when=" argument' + ) + if "when" in kwargs: + kwargs["when"] = (*DirectiveMeta._when_constraints_stack, kwargs["when"]) + else: + kwargs["when"] = tuple(DirectiveMeta._when_constraints_stack) + + # Remove directives passed as arguments, so they are not executed as part of this + # class's directive execution, but handled by the called directive instead + if self.can_patch_dependencies and "patches" in kwargs: + DirectiveMeta._remove_kwarg_value_directives_from_queue(kwargs["patches"]) + DirectiveMeta._patches_dependencies = True + + result = decorated_function(*args, **kwargs) + + DirectiveMeta._directives_to_be_executed[decorated_function.__name__].append(result) + + # wrapped function returns same result as original so that we can nest directives + return result + + return _wrapper class DirectiveError(spack.error.SpackError): diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 77b9ac96db1120..d57674716a3f4f 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -283,9 +283,9 @@ def remove_install_directory(self, spec: "spack.spec.Spec", deprecated: bool = F Raised RemoveFailedError if something goes wrong. """ path = self.path_for_spec(spec) - assert path.startswith( - self.root - ), f"Attempted to remove dir outside Spack's install tree. PATH: {path}, ROOT: {self.root}" + assert path.startswith(self.root), ( + "Attempted to remove dir outside Spack's install tree. PATH: {path}, ROOT: {self.root}" + ) if deprecated: if os.path.exists(path): diff --git a/lib/spack/spack/enums.py b/lib/spack/spack/enums.py index ded48034637c69..2bede7f043635b 100644 --- a/lib/spack/spack/enums.py +++ b/lib/spack/spack/enums.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Enumerations used throughout Spack""" + import enum @@ -22,6 +23,8 @@ class ConfigScopePriority(enum.IntEnum): ENVIRONMENT = 2 CUSTOM = 3 COMMAND_LINE = 4 + # Topmost scope reserved for internal use + ENVIRONMENT_SPEC_GROUPS = 5 class PropagationPolicy(enum.Enum): diff --git a/lib/spack/spack/environment/__init__.py b/lib/spack/spack/environment/__init__.py index deb05370c35ef3..fe3c425065ff2d 100644 --- a/lib/spack/spack/environment/__init__.py +++ b/lib/spack/spack/environment/__init__.py @@ -54,6 +54,7 @@ - ``v4`` - ``v5`` - ``v6`` + - ``v7`` * - ``v0.12:0.14`` - ✅ - @@ -61,6 +62,7 @@ - - - + - * - ``v0.15:0.16`` - ✅ - ✅ @@ -68,6 +70,7 @@ - - - + - * - ``v0.17`` - ✅ - ✅ @@ -75,6 +78,7 @@ - - - + - * - ``v0.18:`` - ✅ - ✅ @@ -82,6 +86,7 @@ - ✅ - - + - * - ``v0.22:v0.23`` - ✅ - ✅ @@ -89,7 +94,17 @@ - ✅ - ✅ - - * - ``v1.0:`` + - + * - ``v1.0:1.1`` + - ✅ + - ✅ + - ✅ + - ✅ + - ✅ + - ✅ + - + * - ``v1.2:`` + - ✅ - ✅ - ✅ - ✅ @@ -480,10 +495,11 @@ Version 6 uses specs where compilers are modeled as real dependencies, and not as a node attribute. It doesn't change the top-level lockfile format. -As part of Spack v1.0, compilers stopped being a node attribute, and became a build-only dependency. Packages may -declare a dependency on the c, cxx, or fortran languages, which are now treated as virtuals, and compilers would -be providers for one or more of those languages. Compilers can also inject runtime dependency, on the node being -compiled. The compiler-wrapper is explicitly represented as a node in the DAG, and enters the hash. +As part of Spack v1.0, compilers stopped being a node attribute, and became a build-only +dependency. Packages may declare a dependency on the c, cxx, or fortran languages, which are now +treated as virtuals, and compilers would be providers for one or more of those languages. Compilers +can also inject runtime dependency, on the node being compiled. The compiler-wrapper is explicitly +represented as a node in the DAG, and enters the hash. .. code-block:: json @@ -543,6 +559,40 @@ }, } } + +Version 7 +--------- + +Version 7 adds the additional attribute ``group`` to ``roots``. + +As part of Spack v1.2 each environment can define multiple groups of specs, and fine-tune their +concretization separately. This attribute is needed to associate each root spec with the +corresponding group. + +.. code-block:: json + + { + "_meta": { + "file-type": "spack-lockfile", + "lockfile-version": 7, + "specfile-version": 5 + }, + "spack": { + "version": "1.2.0.dev0", + "type": "git", + "commit": "94b055476f874f424f20e3c0f33b0f22de29220a" + }, + "roots": [ + { + "hash": "o72mlpqvb5xijyqg4iyubpnvd5bfcomb", + "spec": "hdf5", + "group": "default" + } + ], + "concrete_specs": { + } + } + """ from .environment import ( @@ -572,8 +622,10 @@ installed_specs, is_env_dir, is_latest_format, + lockfile_include_key, lockfile_name, manifest_file, + manifest_include_name, manifest_name, no_active_environment, read, @@ -610,8 +662,10 @@ "installed_specs", "is_env_dir", "is_latest_format", + "lockfile_include_key", "lockfile_name", "manifest_file", + "manifest_include_name", "manifest_name", "no_active_environment", "read", diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index dda534ea62a3e3..9d53c34c0a438b 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -12,15 +12,29 @@ import shutil import stat import warnings -from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union +from collections.abc import KeysView +from itertools import zip_longest +from typing import ( + Any, + Callable, + Dict, + Iterable, + List, + Mapping, + Optional, + Sequence, + Set, + Tuple, + Union, +) import spack -import spack.concretize import spack.config import spack.deptypes as dt import spack.error import spack.filesystem_view as fsv import spack.hash_types as ht +import spack.installer_dispatch import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as clr @@ -39,16 +53,20 @@ import spack.util.spack_yaml as syaml import spack.variant as vt from spack import traverse +from spack.enums import ConfigScopePriority from spack.llnl.util.filesystem import copy_tree, islink, readlink, symlink +from spack.llnl.util.lang import stable_partition from spack.llnl.util.link_tree import ConflictingSpecsError from spack.schema.env import TOP_LEVEL_KEY from spack.spec import Spec +from spack.spec_filter import SpecFilter from spack.util.path import substitute_path_variables -from ..enums import ConfigScopePriority from .list import SpecList, SpecListError, SpecListParser -SpecPair = spack.concretize.SpecPair +SpecPair = Tuple[Spec, Spec] + +DEFAULT_USER_SPEC_GROUP = "default" #: environment variable used to indicate the active environment spack_env_var = "SPACK_ENV" @@ -133,9 +151,7 @@ def default_manifest_yaml(): view: true concretizer: unify: {} -""".format( - "true" if spack.config.get("concretizer:unify") else "false" - ) +""".format("true" if spack.config.get("concretizer:unify") else "false") sep_re = re.escape(os.sep) @@ -144,7 +160,7 @@ def default_manifest_yaml(): valid_environment_name_re = rf"^\w[{sep_re}\w-]*$" #: version of the lockfile format. Must increase monotonically. -lockfile_format_version = 6 +CURRENT_LOCKFILE_VERSION = 7 READER_CLS = { @@ -154,18 +170,24 @@ def default_manifest_yaml(): 4: spack.spec.SpecfileV3, 5: spack.spec.SpecfileV4, 6: spack.spec.SpecfileV5, + 7: spack.spec.SpecfileV5, } # Magic names # The name of the standalone spec list in the manifest yaml -user_speclist_name = "specs" +USER_SPECS_KEY = "specs" # The name of the default view (the view loaded on env.activate) default_view_name = "default" # Default behavior to link all packages into views (vs. only root packages) default_view_link = "all" -# The name for any included concrete specs -included_concrete_name = "include_concrete" + +# (DEPRECATED) Use as the heading/name in the manifest is deprecated. +# The key for any concrete specs included in a lockfile. +lockfile_include_key = "include_concrete" + +# The name/heading for include paths in the manifest file. +manifest_include_name = "include" def installed_specs(): @@ -281,7 +303,7 @@ def root(name): def exists(name): """Whether an environment with this name exists or not.""" - return valid_env_name(name) and os.path.isdir(_root(name)) + return valid_env_name(name) and os.path.lexists(os.path.join(_root(name), manifest_name)) def active(name): @@ -302,7 +324,7 @@ def as_env_dir(name_or_dir): validate_env_name(name_or_dir) if not exists(name_or_dir): raise SpackEnvironmentError("no such environment '%s'" % name_or_dir) - return root(name_or_dir) + return _root(name_or_dir) def environment_from_name_or_dir(name_or_dir): @@ -340,7 +362,7 @@ def create( string, it specifies the path to the view keep_relative: if True, develop paths are copied verbatim into the new environment file, otherwise they are made absolute - include_concrete: list of concrete environment names/paths to be included + include_concrete: concrete environment names/paths to be included """ environment_dir = environment_dir_from_name(name, exists_ok=False) return create_in_dir( @@ -561,8 +583,8 @@ def validate_included_envs_concrete(include_concrete: List[str]) -> None: non_concrete_envs = set() for env_path in include_concrete: - if not os.path.exists(Environment(env_path).lock_path): - non_concrete_envs.add(Environment(env_path).name) + if not os.path.exists(os.path.join(env_path, lockfile_name)): + non_concrete_envs.add(environment_name(env_path)) if non_concrete_envs: msg = "The following environment(s) are not concrete: {0}\nPlease run:".format( @@ -674,43 +696,50 @@ def _error_on_nonempty_view_dir(new_root): class ViewDescriptor: def __init__( self, - base_path, - root, - projections={}, - select=[], - exclude=[], - link=default_view_link, - link_type="symlink", - ): + base_path: str, + root: str, + *, + projections: Optional[Dict[str, str]] = None, + select: Optional[List[str]] = None, + exclude: Optional[List[str]] = None, + link: str = default_view_link, + link_type: fsv.LinkType = "symlink", + link_dirs: bool = True, + groups: Optional[Union[str, List[str]]] = None, + ) -> None: self.base = base_path self.raw_root = root self.root = spack.util.path.canonicalize_path(root, default_wd=base_path) - self.projections = projections - self.select = select - self.exclude = exclude - self.link_type = fsv.canonicalize_link_type(link_type) + self.projections = projections or {} + self.select = select or [] + self.exclude = exclude or [] + self.link_type: fsv.LinkType = fsv.canonicalize_link_type(link_type) + self.link_dirs: bool = link_type == "symlink" and link_dirs self.link = link + if isinstance(groups, str): + groups = [groups] + self.groups: Optional[List[str]] = groups - def select_fn(self, spec): + def select_fn(self, spec: Spec) -> bool: return any(spec.satisfies(s) for s in self.select) - def exclude_fn(self, spec): + def exclude_fn(self, spec: Spec) -> bool: return not any(spec.satisfies(e) for e in self.exclude) - def update_root(self, new_path): + def update_root(self, new_path: str) -> None: self.raw_root = new_path self.root = spack.util.path.canonicalize_path(new_path, default_wd=self.base) - def __eq__(self, other): - return all( - [ - self.root == other.root, - self.projections == other.projections, - self.select == other.select, - self.exclude == other.exclude, - self.link == other.link, - self.link_type == other.link_type, - ] + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, ViewDescriptor) + and self.root == other.root + and self.projections == other.projections + and self.select == other.select + and self.exclude == other.exclude + and self.link == other.link + and self.link_type == other.link_type + and self.link_dirs == other.link_dirs ) def to_dict(self): @@ -723,24 +752,28 @@ def to_dict(self): ret["exclude"] = self.exclude if self.link_type: ret["link_type"] = self.link_type + if self.link_dirs: + ret["link_dirs"] = self.link_dirs if self.link != default_view_link: ret["link"] = self.link return ret @staticmethod - def from_dict(base_path, d): + def from_dict(base_path: str, d) -> "ViewDescriptor": return ViewDescriptor( base_path, d["root"], - d.get("projections", {}), - d.get("select", []), - d.get("exclude", []), - d.get("link", default_view_link), - d.get("link_type", "symlink"), + projections=d.get("projections", {}), + select=d.get("select", []), + exclude=d.get("exclude", []), + link=d.get("link", default_view_link), + link_type=d.get("link_type", "symlink"), + link_dirs=d.get("link_dirs", True), + groups=d.get("group", None), ) @property - def _current_root(self): + def _current_root(self) -> Optional[str]: if not islink(self.root): return None @@ -764,7 +797,7 @@ def content_hash(self, specs): ("specs", [(spec.dag_hash(), spec.prefix) for spec in sorted(specs)]), ] ) - contents = sjson.dump(d) + contents = sjson.dumps(d) return spack.util.hash.b32_hash(contents) def get_projection_for_spec(self, spec): @@ -800,6 +833,7 @@ def _view(self, root: str) -> fsv.SimpleFilesystemView: ignore_conflicts=True, projections=self.projections, link_type=self.link_type, + link_dirs=self.link_dirs, ) def __contains__(self, spec): @@ -839,7 +873,12 @@ def specs_for_view(self, concrete_roots: List[Spec]) -> List[Spec]: return self._exclude_duplicate_runtimes(result) - def regenerate(self, concrete_roots: List[Spec]) -> None: + def regenerate(self, env: "Environment") -> None: + if self.groups is None: + concrete_roots = env.concrete_roots() + else: + concrete_roots = [c for g in self.groups for _, c in env.concretized_specs_by(group=g)] + specs = self.specs_for_view(concrete_roots) # To ensure there are no conflicts with packages being installed @@ -941,15 +980,17 @@ def regenerate(self, concrete_roots: List[Spec]) -> None: msg += str(e) tty.warn(msg) - def _exclude_duplicate_runtimes(self, nodes): - all_runtimes = spack.repo.PATH.packages_with_tags("runtime") - runtimes_by_name = {} - for s in nodes: - if s.name not in all_runtimes: + def _exclude_duplicate_runtimes(self, specs: List[Spec]) -> List[Spec]: + """Stably filter out duplicates of "runtime" tagged packages, keeping only latest.""" + # Maps packages tagged "runtime" to the spec with latest version. + latest: Dict[str, Spec] = {} + for s in specs: + if "runtime" not in getattr(s.package, "tags", ()): continue - current_runtime = runtimes_by_name.get(s.name, s) - runtimes_by_name[s.name] = max(current_runtime, s, key=lambda x: x.version) - return [x for x in nodes if x.name not in all_runtimes or runtimes_by_name[x.name] == x] + elif s.name not in latest or latest[s.name].version < s.version: + latest[s.name] = s + + return [x for x in specs if x.name not in latest or latest[x.name] is x] def env_subdir_path(manifest_dir: Union[str, pathlib.Path]) -> str: @@ -963,6 +1004,45 @@ def env_subdir_path(manifest_dir: Union[str, pathlib.Path]) -> str: return os.path.join(str(manifest_dir), env_subdir_name) +class ConcretizedRootInfo: + """Data on root specs that have been concretized""" + + __slots__ = ("root", "hash", "new", "group") + + def __init__( + self, *, root_spec: spack.spec.Spec, root_hash: str, new: bool = False, group: str + ): + self.root = root_spec + self.hash = root_hash + self.new = new + self.group = group + + def __str__(self): + return f"{self.root} -> {self.hash} [new={self.new}]" + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, ConcretizedRootInfo) + and self.root == other.root + and self.hash == other.hash + and self.new == other.new + and self.group == other.group + ) + + def __hash__(self) -> int: + return hash((self.root, self.hash, self.new, self.group)) + + @staticmethod + def from_info_dict(info_dict: Dict[str, str]) -> "ConcretizedRootInfo": + # Lockfile versions < 7 don't have the "group" attribute + return ConcretizedRootInfo( + root_spec=Spec(info_dict["spec"]), + root_hash=info_dict["hash"], + new=False, + group=info_dict.get("group", DEFAULT_USER_SPEC_GROUP), + ) + + class Environment: """A Spack environment, which bundles together configuration and a list of specs.""" @@ -980,30 +1060,25 @@ def __init__(self, manifest_dir: Union[str, pathlib.Path]) -> None: self.txlock = lk.Lock(self._transaction_lock_path) self._unify = None - self.new_specs: List[Spec] = [] self.views: Dict[str, ViewDescriptor] = {} #: Parser for spec lists self._spec_lists_parser = SpecListParser() #: Specs from "spack.yaml" self.spec_lists: Dict[str, SpecList] = {} - #: User specs from the last concretization - self.concretized_user_specs: List[Spec] = [] - #: Roots associated with the last concretization, in order - self.concretized_order: List[str] = [] + #: Information on concretized roots + self.concretized_roots: List[ConcretizedRootInfo] = [] #: Concretized specs by hash self.specs_by_hash: Dict[str, Spec] = {} #: Repository for this environment (memoized) self._repo = None - #: Environment paths for concrete (lockfile) included environments - self.included_concrete_envs: List[str] = [] + #: Environment root dirs for concrete (lockfile) included environments + self.included_concrete_env_root_dirs: List[str] = [] #: First-level included concretized spec data from/to the lockfile. self.included_concrete_spec_data: Dict[str, Dict[str, List[str]]] = {} - #: User specs from included environments from the last concretization - self.included_concretized_user_specs: Dict[str, List[Spec]] = {} - #: Roots from included environments with the last concretization, in order - self.included_concretized_order: Dict[str, List[str]] = {} + #: Roots from included environments from the last concretization, keyed by env path + self.included_concretized_roots: Dict[str, List[ConcretizedRootInfo]] = {} #: Concretized specs by hash from the included environments self.included_specs_by_hash: Dict[str, Dict[str, Spec]] = {} @@ -1021,15 +1096,30 @@ def _load_manifest_file(self): with self.manifest.use_config(): self._read() - @property - def unify(self): - if self._unify is None: - self._unify = spack.config.get("concretizer:unify", False) - return self._unify + @contextlib.contextmanager + def config_override_for_group(self, *, group: str): + key = self.manifest._ensure_group_exists(group=group) + internal_scope = self.manifest.config_override(group=key) + if internal_scope is None: + # No internal scope + tty.debug( + f"[{__name__}] No configuration override necessary for the '{group}' group " + f"in the environment at {self.manifest_path}" + ) + yield + return - @unify.setter - def unify(self, value): - self._unify = value + try: + tty.debug( + f"[{__name__}] Overriding the configuration for the '{group}' group defined " + f"in {self.manifest_path} before concretization" + ) + spack.config.CONFIG.push_scope( + internal_scope, priority=ConfigScopePriority.ENVIRONMENT_SPEC_GROUPS + ) + yield + finally: + spack.config.CONFIG.remove_scope(internal_scope.name) def __getstate__(self): state = self.__dict__.copy() @@ -1046,7 +1136,7 @@ def __setstate__(self, state): def _re_read(self): """Reinitialize the environment object.""" - self.clear(re_read=True) + self.clear() self._load_manifest_file() def _read(self): @@ -1109,72 +1199,104 @@ def add_view(name, values): if self.views == dict(): self.views[default_view_name] = ViewDescriptor(self.path, self.view_path_default) - def _process_concrete_includes(self): - """Extract and load into memory included concrete spec data.""" - _included_concrete_envs = self.manifest[TOP_LEVEL_KEY].get(included_concrete_name, []) - # Expand config and environment variables - self.included_concrete_envs = [ - spack.util.path.canonicalize_path(_env) for _env in _included_concrete_envs - ] - - if self.included_concrete_envs: + def _load_concrete_include_data(self): + """Load concrete include specs data from included concrete directories.""" + if self.included_concrete_env_root_dirs: if os.path.exists(self.lock_path): with open(self.lock_path, encoding="utf-8") as f: data = self._read_lockfile(f) - if included_concrete_name in data: - self.included_concrete_spec_data = data[included_concrete_name] + if lockfile_include_key in data: + self.included_concrete_spec_data = data[lockfile_include_key] else: self.include_concrete_envs() + def _process_included_lockfiles(self): + """Extract and load into memory included lock file data.""" + includes = self.manifest[TOP_LEVEL_KEY].get(lockfile_include_key, []) + if includes: + tty.warn( + f"Use of '{lockfile_include_key}' in manifest files " + f"is deprecated. The key should be '{manifest_include_name}' " + f"and the path should end with '{lockfile_name}'. Run " + f"'spack env update {self.name}' to update the manifest." + ) + includes = [os.path.join(inc, lockfile_name) for inc in includes] + includes += self.manifest[TOP_LEVEL_KEY].get(manifest_include_name, []) + if not includes: + return + + # Expand config and environment variables for concrete environments, + # indicated by the inclusion of lock files. + self.included_concrete_env_root_dirs = [] + + for entry in includes: + include = spack.config.included_path(entry) + if isinstance(include, spack.config.GitIncludePaths): + # Git includes must be cloned first; paths are relative to the + # clone destination, not to the manifest directory. + destination = include._clone(self.manifest.env_config_scope) + if destination is None: + continue + resolved = [os.path.join(destination, p) for p in include.paths] + else: + resolved = [ + spack.util.path.canonicalize_path(p, default_wd=self.path) + for p in include.paths + ] + + for path in resolved: + if os.path.basename(path) != lockfile_name: + continue + + tty.debug(f"Adding {path} to the concrete environment root directories") + self.included_concrete_env_root_dirs.append(os.path.dirname(path)) + + # Cache concrete environments for required lock files. + self._load_concrete_include_data() + def _construct_state_from_manifest(self): """Set up user specs and views from the manifest file.""" self.views = {} self._sync_speclists() self._process_view(spack.config.get("view", True)) - self._process_concrete_includes() + self._process_included_lockfiles() def _sync_speclists(self): + self._spec_lists_parser = SpecListParser( + toolchains=spack.config.CONFIG.get("toolchains", {}) + ) self.spec_lists = {} self.spec_lists.update( self._spec_lists_parser.parse_definitions( data=spack.config.CONFIG.get("definitions", []) ) ) + for group in self.manifest.groups(): + tty.debug(f"[{__name__}]: Synchronizing user specs from the '{group}' group", level=2) + key = self._user_specs_key(group=group) + self.spec_lists[key] = self._spec_lists_parser.parse_user_specs( + name=key, yaml_list=self.manifest.user_specs(group=group) + ) - env_configuration = self.manifest[TOP_LEVEL_KEY] - spec_list = env_configuration.get(user_speclist_name, []) - self.spec_lists[user_speclist_name] = self._spec_lists_parser.parse_user_specs( - name=user_speclist_name, yaml_list=spec_list - ) - - def all_concretized_user_specs(self) -> List[Spec]: - """Returns all of the concretized user specs of the environment and - its included environment(s).""" - concretized_user_specs = self.concretized_user_specs[:] - for included_specs in self.included_concretized_user_specs.values(): - for included in included_specs: - # Don't duplicate included spec(s) - if included not in concretized_user_specs: - concretized_user_specs.append(included) - - return concretized_user_specs - - def all_concretized_orders(self) -> List[str]: - """Returns all of the concretized order of the environment and - its included environment(s).""" - concretized_order = self.concretized_order[:] - for included_concretized_order in self.included_concretized_order.values(): - for included in included_concretized_order: - # Don't duplicate included spec(s) - if included not in concretized_order: - concretized_order.append(included) - - return concretized_order + def _user_specs_key(self, *, group: Optional[str] = None) -> str: + if group is None or group == DEFAULT_USER_SPEC_GROUP: + return USER_SPECS_KEY + return f"{USER_SPECS_KEY}:{group}" @property - def user_specs(self): - return self.spec_lists[user_speclist_name] + def user_specs(self) -> SpecList: + return self.user_specs_by(group=DEFAULT_USER_SPEC_GROUP) + + def user_specs_by(self, *, group: Optional[str]) -> SpecList: + """Returns a dictionary of user specs keyed by their group.""" + key = self._user_specs_key(group=group) + return self.spec_lists[key] + + def explicit_roots(self): + for x in self.concretized_roots: + if self.manifest.is_explicit(group=x.group): + yield x @property def dev_specs(self): @@ -1195,7 +1317,7 @@ def included_user_specs(self) -> SpecList: """Included concrete user (or root) specs from last concretization.""" spec_list = SpecList() - if not self.included_concrete_envs: + if not self.included_concrete_env_root_dirs: return spec_list def add_root_specs(included_concrete_specs): @@ -1204,35 +1326,25 @@ def add_root_specs(included_concrete_specs): for root_list in info["roots"]: spec_list.add(root_list["spec"]) - if "include_concrete" in info: - add_root_specs(info["include_concrete"]) + if lockfile_include_key in info: + add_root_specs(info[lockfile_include_key]) add_root_specs(self.included_concrete_spec_data) return spec_list - def clear(self, re_read=False): - """Clear the contents of the environment - - Arguments: - re_read: If ``True``, do not clear ``new_specs``. This value cannot be read from yaml, - and needs to be maintained when re-reading an existing environment. - """ + def clear(self): + """Clear the contents of the environment""" self.spec_lists = {} self._dev_specs = {} - self.concretized_order = [] # roots of last concretize, in order - self.concretized_user_specs = [] # user specs from last concretize + self.concretized_roots = [] self.specs_by_hash = {} # concretized specs by hash self.included_concrete_spec_data = {} # concretized specs from lockfile of included envs - self.included_concretized_order = {} # root specs of the included envs, keyed by env path - self.included_concretized_user_specs = {} # user specs from last concretize's included env + self.included_concretized_roots = {} # root specs of the included envs, keyed by env path self.included_specs_by_hash = {} # concretized specs by hash from the included envs self.invalidate_repository_cache() self._previous_active = None # previously active environment - if not re_read: - # things that cannot be recreated from file - self.new_specs = [] # write packages for these on write() self.manifest.clear() @@ -1268,7 +1380,7 @@ def repos_path(self): return os.path.join(self.env_subdir_path, "repos") @property - def view_path_default(self): + def view_path_default(self) -> str: # default path for environment views return os.path.join(self.env_subdir_path, "view") @@ -1284,14 +1396,14 @@ def scope_name(self): return self.manifest.scope_name def include_concrete_envs(self): - """Copy and save the included envs' specs internally""" + """Copy and save the included environments' specs internally.""" root_hash_seen = set() concrete_hash_seen = set() self.included_concrete_spec_data = {} - for env_path in self.included_concrete_envs: - # Check that environment exists + for env_path in self.included_concrete_env_root_dirs: + # Check that the environment (lockfile) exists if not is_env_dir(env_path): raise SpackEnvironmentError(f"Unable to find env at {env_path}") @@ -1315,16 +1427,16 @@ def include_concrete_envs(self): # Copy transitive include data transitive = env.included_concrete_spec_data if transitive: - self.included_concrete_spec_data[env_path]["include_concrete"] = transitive + self.included_concrete_spec_data[env_path][lockfile_include_key] = transitive - self._read_lockfile_dict(self._to_lockfile_dict()) + self.unify_specs() self.write() def destroy(self): """Remove this environment from Spack entirely.""" shutil.rmtree(self.path) - def add(self, user_spec, list_name=user_speclist_name) -> bool: + def add(self, user_spec, list_name=USER_SPECS_KEY) -> bool: """Add a single user_spec (non-concretized) to the Environment Returns: @@ -1337,7 +1449,7 @@ def add(self, user_spec, list_name=user_speclist_name) -> bool: if list_name not in self.spec_lists: raise SpackEnvironmentError(f"No list {list_name} exists in environment {self.name}") - if list_name == user_speclist_name: + if list_name == USER_SPECS_KEY: if spec.anonymous: raise SpackEnvironmentError("cannot add anonymous specs to an environment") elif not spack.repo.PATH.exists(spec.name) and not spec.abstract_hash: @@ -1349,7 +1461,7 @@ def add(self, user_spec, list_name=user_speclist_name) -> bool: existing = str(spec) in list_to_change.yaml_list if not existing: list_to_change.add(spec) - if list_name == user_speclist_name: + if list_name == USER_SPECS_KEY: self.manifest.add_user_spec(str(user_spec)) else: self.manifest.add_definition(str(user_spec), list_name=list_name) @@ -1360,7 +1472,7 @@ def add(self, user_spec, list_name=user_speclist_name) -> bool: def change_existing_spec( self, change_spec: Spec, - list_name: str = user_speclist_name, + list_name: str = USER_SPECS_KEY, match_spec: Optional[Spec] = None, allow_changing_multiple_specs=False, ): @@ -1401,7 +1513,7 @@ def change_existing_spec( for idx, spec in matches: override_spec = Spec.override(spec, change_spec) - if list_name == user_speclist_name: + if list_name == USER_SPECS_KEY: self.manifest.override_user_spec(str(override_spec), idx=idx) else: self.manifest.override_definition( @@ -1409,7 +1521,7 @@ def change_existing_spec( ) self._sync_speclists() - def remove(self, query_spec, list_name=user_speclist_name, force=False): + def remove(self, query_spec, list_name=USER_SPECS_KEY, force=False): """Remove specs from an environment that match a query_spec""" err_msg_header = ( f"Cannot remove '{query_spec}' from '{list_name}' definition " @@ -1426,10 +1538,8 @@ def remove(self, query_spec, list_name=user_speclist_name, force=False): matches = [s for s in list_to_change if s.satisfies(query_spec)] else: - # concrete specs match against concrete specs in the env - # by dag hash. - specs_hashes = zip(self.concretized_user_specs, self.concretized_order) - matches = [s for s, h in specs_hashes if query_spec.dag_hash() == h] + # concrete specs match against concrete specs in the env by dag hash. + matches = [x.root for x in self.concretized_roots if query_spec.dag_hash() == x.hash] if not matches: raise SpackEnvironmentError(f"{err_msg_header}, no spec matches") @@ -1448,7 +1558,7 @@ def remove(self, query_spec, list_name=user_speclist_name, force=False): msg += " It will be removed from the concrete specs." tty.warn(msg) else: - if list_name == user_speclist_name: + if list_name == USER_SPECS_KEY: self.manifest.remove_user_spec(str(spec)) else: self.manifest.remove_definition(str(spec), list_name=list_name) @@ -1458,47 +1568,57 @@ def remove(self, query_spec, list_name=user_speclist_name, force=False): new_specs = set(self.user_specs) # If 'force', update stale concretized specs - for spec in old_specs - new_specs: - if force and spec in self.concretized_user_specs: - i = self.concretized_user_specs.index(spec) - del self.concretized_user_specs[i] - - dag_hash = self.concretized_order[i] - del self.concretized_order[i] - del self.specs_by_hash[dag_hash] + if force: + stale_specs = old_specs - new_specs + self.concretized_roots, removed = stable_partition( + self.concretized_roots, lambda x: x.root not in stale_specs + ) + for x in removed: + del self.specs_by_hash[x.hash] def is_develop(self, spec): """Returns true when the spec is built from local sources""" return spec.name in self.dev_specs - def apply_develop(self, spec: spack.spec.Spec, path: Optional[str] = None): + def apply_develop(self, specs: List[spack.spec.Spec], paths: Optional[List[str]] = None): """Mutate concrete specs to include dev_path provenance pointing to path. This will fail if any existing concrete spec for the same package does not satisfy the given develop spec.""" - selector = spack.spec.Spec(spec.name) + selectors = [] + mutators = [] + msgs = [] + + assert not paths or len(specs) == len(paths) + for spec, path in zip_longest(specs, paths or [], fillvalue=None): + assert spec + selector = spack.spec.Spec(spec.name) + + mutator = spack.spec.Spec() + if path: + variant = vt.SingleValuedVariant("dev_path", path) + else: + variant = vt.VariantValueRemoval("dev_path") + mutator.variants["dev_path"] = variant - mutator = spack.spec.Spec() - if path: - variant = vt.SingleValuedVariant("dev_path", path) - else: - variant = vt.VariantValueRemoval("dev_path") - mutator.variants["dev_path"] = variant + msg = ( + f"Develop spec '{spec}' conflicts with concrete specs in environment." + " Try again with 'spack develop --no-modify-concrete-specs'" + " and run 'spack concretize --force' to apply your changes." + ) + selectors.append(selector) + mutators.append(mutator) + msgs.append(msg) - msg = ( - f"Develop spec '{spec}' conflicts with concrete specs in environment." - " Try again with 'spack develop --no-modify-concrete-specs'" - " and run 'spack concretize --force' to apply your changes." - ) - self.mutate(selector, mutator, validator=spec, msg=msg) + self.mutate(selectors, mutators, validators=specs, msgs=msgs) def mutate( self, - selector: spack.spec.Spec, - mutator: spack.spec.Spec, - validator: Optional[spack.spec.Spec] = None, - msg: Optional[str] = None, + selectors: List[spack.spec.Spec], + mutators: List[spack.spec.Spec], + validators: Optional[List[spack.spec.Spec]] = None, + msgs: Optional[List[str]] = None, ): """Mutate concrete specs of an environment @@ -1509,17 +1629,37 @@ def mutate( # Find all specs that this mutation applies to modify_specs = [] modified_specs = [] + if len(selectors) != len(mutators): + raise ValueError( + f"Length mismatch: selectors ({len(selectors)}) != mutators ({len(mutators)})" + ) + + if validators and len(validators) != len(selectors): + raise ValueError( + f"Length mismatch: validators ({len(validators)}) != selectors ({len(selectors)})" + ) + + if msgs and len(msgs) != len(selectors): + raise ValueError( + f"Length mismatch: msgs ({len(msgs)}) != selectors ({len(selectors)})" + ) + for dep in self.all_specs_generator(): - if dep.satisfies(selector): - if not dep.satisfies(validator or selector): - if not msg: - msg = f"spec {dep} satisfies selector {selector}" - msg += f" but not validator {validator}" - raise SpackEnvironmentDevelopError(msg) - modify_specs.append(dep) + for selector, mutator, validator, msg in zip_longest( + selectors, mutators, validators or [], msgs or [], fillvalue=None + ): + assert selector + assert mutator + if dep.satisfies(selector): + if not dep.satisfies(validator or selector): + if not msg: + msg = f"spec {dep} satisfies selector {selector}" + msg += f" but not validator {validator}" + raise SpackEnvironmentDevelopError(msg) + modify_specs.append((dep, mutator)) # Manipulate selected specs - for s in modify_specs: + for s, mutator in modify_specs: modified = s.mutate(mutator, rehash=False) if modified: modified_specs.append(s) @@ -1535,25 +1675,31 @@ def mutate( parent.clear_caches() # Compute new hashes and update the env list of specs + hash_mutations = {} for root, old_hash in modified_roots: + # New hash must be computed after we finalize concretization root._finalize_concretization() - self.concretized_order[self.concretized_order.index(old_hash)] = root.dag_hash() + new_hash = root.dag_hash() self.specs_by_hash.pop(old_hash) - self.specs_by_hash[root.dag_hash()] = root + self.specs_by_hash[new_hash] = root + hash_mutations[old_hash] = new_hash + + for x in self.concretized_roots: + if x.hash in hash_mutations: + x.hash = hash_mutations[x.hash] if modified_roots: self.write() def concretize( - self, force: Optional[bool] = None, tests: Union[bool, Sequence] = False + self, *, force: Optional[bool] = None, tests: Union[bool, Sequence[str]] = False ) -> Sequence[SpecPair]: """Concretize user_specs in this environment. - Only concretizes specs that haven't been concretized yet unless - force is ``True``. + Only concretizes specs that haven't been concretized yet unless force is ``True``. - This only modifies the environment in memory. ``write()`` will - write out a lockfile containing concretized specs. + This only modifies the environment in memory. ``write()`` will write out a lockfile + containing concretized specs. Arguments: force: re-concretize ALL specs, even those that were already concretized; @@ -1565,191 +1711,72 @@ def concretize( List of specs that have been concretized. Each entry is a tuple of the user spec and the corresponding concretized spec. """ - if force is None: - force = spack.config.get("concretizer:force") - - if force: - # Clear previously concretized specs - self.concretized_user_specs = [] - self.concretized_order = [] - self.specs_by_hash = {} + return EnvironmentConcretizer(self).concretize(force=force, tests=tests) - # Remove concrete specs that no longer correlate to a user spec - for spec in set(self.concretized_user_specs) - set(self.user_specs): - self.deconcretize(spec, concrete=False) - - # If a combined env, check updated spec is in the linked envs - if self.included_concrete_envs: - self.include_concrete_envs() - - # Pick the right concretization strategy - if self.unify == "when_possible": - return self._concretize_together_where_possible(tests=tests) + def sync_concretized_specs(self) -> None: + """Removes concrete specs that no longer correlate to a user spec""" + if not self.concretized_roots: + return - if self.unify is True: - return self._concretize_together(tests=tests) + to_deconcretize, user_specs = [], self._all_user_specs_with_group() + for x in self.concretized_roots: + if (x.group, x.root) not in user_specs: + to_deconcretize.append(x) + for x in to_deconcretize: + self.deconcretize_by_user_spec(x.root, group=x.group) + + def _all_user_specs_with_group(self) -> Set[Tuple[str, Spec]]: + result = set() + for group in self.manifest.groups(): + result.update([(group, x) for x in self.user_specs_by(group=group)]) + return result - if self.unify is False: - return self._concretize_separately(tests=tests) + def clear_concretized_specs(self) -> None: + """Clears the currently concretized specs""" + self.concretized_roots = [] + self.specs_by_hash = {} - msg = "concretization strategy not implemented [{0}]" - raise SpackEnvironmentError(msg.format(self.unify)) + def deconcretize_by_hash(self, dag_hash: str) -> None: + """Removes a concrete spec from the environment concretization""" + self.concretized_roots = [x for x in self.concretized_roots if x.hash != dag_hash] + self._maybe_remove_dag_hash(dag_hash) - def deconcretize(self, spec: spack.spec.Spec, concrete: bool = True): - """ - Remove specified spec from environment concretization + def deconcretize_by_user_spec( + self, spec: spack.spec.Spec, *, group: Optional[str] = None + ) -> None: + """Removes a user spec from the environment concretization Arguments: - spec: Spec to deconcretize. This must be a root of the environment - concrete: If True, find all instances of spec as concrete in the environment. - If False, find a single instance of the abstract spec as root of the environment. + spec: user spec to deconcretize + group: group of the spec to remove. If not specified, the spec is removed from + the default group """ + group = group or DEFAULT_USER_SPEC_GROUP # spec has to be a root of the environment - if concrete: - dag_hash = spec.dag_hash() - - pairs = zip(self.concretized_user_specs, self.concretized_order) - filtered = [(spec, h) for spec, h in pairs if h != dag_hash] - # Cannot use zip and unpack two values; it fails if filtered is empty - self.concretized_user_specs = [s for s, _ in filtered] - self.concretized_order = [h for _, h in filtered] - else: - index = self.concretized_user_specs.index(spec) - dag_hash = self.concretized_order.pop(index) - - del self.concretized_user_specs[index] - - # If this was the only user spec that concretized to this concrete spec, remove it - if dag_hash not in self.concretized_order: - # if we deconcretized a dependency that doesn't correspond to a root, it - # won't be here. - if dag_hash in self.specs_by_hash: - del self.specs_by_hash[dag_hash] - - def _get_specs_to_concretize( - self, - ) -> Tuple[List[spack.spec.Spec], List[spack.spec.Spec], List[SpecPair]]: - """Compute specs to concretize for unify:true and unify:when_possible. - - This includes new user specs and any already concretized specs. - - Returns: - Tuple of new user specs, user specs to keep, and the specs to concretize. - - """ - # Exit early if the set of concretized specs is the set of user specs - new_user_specs = list(set(self.user_specs) - set(self.concretized_user_specs)) - kept_user_specs = list(set(self.user_specs) & set(self.concretized_user_specs)) - kept_user_specs += self.included_user_specs - if not new_user_specs: - return new_user_specs, kept_user_specs, [] - - specs_to_concretize = [(s, None) for s in new_user_specs] + [ - (abstract, concrete) - for abstract, concrete in self.concretized_specs() - if abstract in kept_user_specs - ] - return new_user_specs, kept_user_specs, specs_to_concretize - - def _concretize_together_where_possible( - self, tests: Union[bool, Sequence] = False - ) -> Sequence[SpecPair]: - # Exit early if the set of concretized specs is the set of user specs - new_user_specs, _, specs_to_concretize = self._get_specs_to_concretize() - if not new_user_specs: - return [] - - self.concretized_user_specs = [] - self.concretized_order = [] - self.specs_by_hash = {} - - ret = [] - result = spack.concretize.concretize_together_when_possible( - specs_to_concretize, tests=tests + discarded, self.concretized_roots = stable_partition( + self.concretized_roots, lambda x: x.group == group and x.root == spec ) - for abstract, concrete in result: - # Only add to the environment if it's from this environment (not included in) - if abstract in self.user_specs: - self._add_concrete_spec(abstract, concrete) - - # Return only the new specs - if abstract in new_user_specs: - ret.append((abstract, concrete)) - - return ret - - def _concretize_together(self, tests: Union[bool, Sequence] = False) -> Sequence[SpecPair]: - """Concretization strategy that concretizes all the specs - in the same DAG. - """ - # Exit early if the set of concretized specs is the set of user specs - new_user_specs, kept_user_specs, specs_to_concretize = self._get_specs_to_concretize() - if not new_user_specs: - return [] - - self.concretized_user_specs = [] - self.concretized_order = [] - self.specs_by_hash = {} - - try: - concretized_specs = spack.concretize.concretize_together( - specs_to_concretize, tests=tests - ) - except spack.error.UnsatisfiableSpecError as e: - # "Enhance" the error message for multiple root specs, suggest a less strict - # form of concretization. - if len(self.user_specs) > 1: - e.message += ". " - if kept_user_specs: - e.message += ( - "Couldn't concretize without changing the existing environment. " - "If you are ok with changing it, try `spack concretize --force`. " - ) - e.message += ( - "You could consider setting `concretizer:unify` to `when_possible` " - "or `false` to allow multiple versions of some packages." - ) - raise - - for abstract, concrete in concretized_specs: - # Don't add if it's just included - if abstract in self.user_specs: - self._add_concrete_spec(abstract, concrete) - - # Return the portion of the return value that is new - return concretized_specs[: len(new_user_specs)] - - def _concretize_separately(self, tests: Union[bool, Sequence] = False): - """Concretization strategy that concretizes separately one - user spec after the other. - """ - # keep any concretized specs whose user specs are still in the manifest - old_concretized_user_specs = self.concretized_user_specs - old_concretized_order = self.concretized_order - old_specs_by_hash = self.specs_by_hash - - self.concretized_user_specs = [] - self.concretized_order = [] - self.specs_by_hash = {} - - for s, h in zip(old_concretized_user_specs, old_concretized_order): - if s in self.user_specs: - concrete = old_specs_by_hash[h] - self._add_concrete_spec(s, concrete, new=False) + assert len({x.hash for x in discarded}) == 1, ( + "More than one hash associated with a single user spec" + ) + dag_hash = discarded[0].hash + self._maybe_remove_dag_hash(dag_hash) - to_concretize = [ - (root, None) for root in self.user_specs if root not in old_concretized_user_specs - ] - concretized_specs = spack.concretize.concretize_separately(to_concretize, tests=tests) + def _maybe_remove_dag_hash(self, dag_hash: str): + # If this was the only user spec that concretized to this concrete spec, remove it + if not self.user_spec_with_hash(dag_hash) and dag_hash in self.specs_by_hash: + # if we deconcretized a dependency that doesn't correspond to a root, it won't be here. + del self.specs_by_hash[dag_hash] - by_hash = {} - for abstract, concrete in concretized_specs: - self._add_concrete_spec(abstract, concrete) - by_hash[concrete.dag_hash()] = concrete + def user_spec_with_hash(self, dag_hash: str) -> bool: + """Returns True if any user spec is associated with a concrete spec with the given hash""" + return any(x.hash == dag_hash for x in self.concretized_roots) - # Unify the specs objects, so we get correct references to all parents + def unify_specs(self) -> None: + # Keep the information on new specs by copying the concretized roots + old_concretized_roots = self.concretized_roots self._read_lockfile_dict(self._to_lockfile_dict()) - return concretized_specs + self.concretized_roots = old_concretized_roots @property def default_view(self): @@ -1803,6 +1830,7 @@ def update_default_view(self, path_or_bool: Union[str, bool]) -> None: if default_view_name in self.views: self.default_view.update_root(view_path) else: + assert isinstance(view_path, str), f"expected str for 'view_path', but got {view_path}" self.views[default_view_name] = ViewDescriptor(self.path, view_path) self.manifest.set_default_view(self._default_view_as_yaml()) @@ -1826,7 +1854,7 @@ def regenerate_views(self): return for view in self.views.values(): - view.regenerate(self.concrete_roots()) + view.regenerate(self) def check_views(self): """Checks if the environments default view can be activated.""" @@ -1902,29 +1930,29 @@ def rm_view_from_env( return env_mod - def _add_concrete_spec(self, spec, concrete, new=True): + def add_concrete_spec( + self, + spec: spack.spec.Spec, + concrete: spack.spec.Spec, + *, + new: bool = True, + group: Optional[str] = None, + ): """Called when a new concretized spec is added to the environment. This ensures that all internal data structures are kept in sync. Arguments: - spec (Spec): user spec that resulted in the concrete spec - concrete (Spec): spec concretized within this environment - new (bool): whether to write this spec's package to the env - repo on write() + spec: user spec that resulted in the concrete spec + concrete: spec concretized within this environment + new: whether to write this spec's package to the env repo on write() """ assert concrete.concrete - - # when a spec is newly concretized, we need to make a note so - # that we can write its package to the env repo on write() - if new: - self.new_specs.append(concrete) - - # update internal lists of specs - self.concretized_user_specs.append(spec) - h = concrete.dag_hash() - self.concretized_order.append(h) + group = group or DEFAULT_USER_SPEC_GROUP + self.concretized_roots.append( + ConcretizedRootInfo(root_spec=spec, root_hash=h, new=new, group=group) + ) self.specs_by_hash[h] = concrete def _dev_specs_that_need_overwrite(self): @@ -1958,22 +1986,8 @@ def _partition_roots_by_install_status(self): installer, and those that should be, taking into account development specs. This is done in a single read transaction per environment instead of per spec.""" - installed, uninstalled = [], [] with spack.store.STORE.db.read_transaction(): - for concretized_hash in self.all_concretized_orders(): - if concretized_hash in self.specs_by_hash: - spec = self.specs_by_hash[concretized_hash] - else: - for env_path in self.included_specs_by_hash.keys(): - if concretized_hash in self.included_specs_by_hash[env_path]: - spec = self.included_specs_by_hash[env_path][concretized_hash] - break - if not spec.installed or ( - spec.satisfies("dev_path=*") or spec.satisfies("^dev_path=*") - ): - uninstalled.append(spec) - else: - installed.append(spec) + uninstalled, installed = stable_partition(self.concrete_roots(), _is_uninstalled) return installed, uninstalled def uninstalled_specs(self): @@ -2006,18 +2020,15 @@ def install_specs(self, specs: Optional[List[Spec]] = None, **install_args): *self._dev_specs_that_need_overwrite(), } - # Only environment roots are marked explicit + # Only environment roots in explicit groups are marked explicit install_args["explicit"] = { *install_args.get("explicit", ()), - *(s.dag_hash() for s in roots), + *(x.hash for x in self.explicit_roots()), } - if spack.config.get("config:installer", "old") == "new": - from spack.new_installer import PackageInstaller - else: - from spack.installer import PackageInstaller # type: ignore[assignment] - - builder = PackageInstaller([spec.package for spec in specs], **install_args) + builder = spack.installer_dispatch.create_installer( + [spec.package for spec in specs], create_reports=reporter is not None, **install_args + ) try: builder.install() @@ -2072,20 +2083,38 @@ def added_specs(self): def concretized_specs(self): """Tuples of (user spec, concrete spec) for all concrete specs.""" - for s, h in zip(self.all_concretized_user_specs(), self.all_concretized_orders()): - if h in self.specs_by_hash: - yield (s, self.specs_by_hash[h]) - else: - for env_path in self.included_specs_by_hash.keys(): - if h in self.included_specs_by_hash[env_path]: - yield (s, self.included_specs_by_hash[env_path][h]) - break + for x in self.concretized_roots: + yield x.root, self.specs_by_hash[x.hash] + + yield from self.concretized_specs_from_all_included_environments() + + def concretized_specs_from_all_included_environments(self): + seen = {(x.root, x.hash) for x in self.concretized_roots} + for included_env in self.included_concretized_roots: + yield from self.concretized_specs_from_included_environment(included_env, _seen=seen) + + def concretized_specs_from_included_environment( + self, included_env: str, *, _seen: Optional[Set[Tuple[spack.spec.Spec, str]]] = None + ): + _seen = set() if _seen is None else _seen + for x in self.included_concretized_roots[included_env]: + if (x.root, x.hash) in _seen: + continue + _seen.add((x.root, x.hash)) + yield x.root, self.included_specs_by_hash[included_env][x.hash] def concrete_roots(self): """Same as concretized_specs, except it returns the list of concrete roots *without* associated user spec""" return [root for _, root in self.concretized_specs()] + def concretized_specs_by(self, *, group: str) -> Iterable[Tuple[Spec, Spec]]: + """Generates all the (abstract, concrete) spec pairs for a given group""" + for x in self.concretized_roots: + if x.group != group: + continue + yield x.root, self.specs_by_hash[x.hash] + def get_by_hash(self, dag_hash: str) -> List[Spec]: # If it's not a partial hash prefix we can early exit early_exit = len(dag_hash) == 32 @@ -2202,22 +2231,6 @@ def removed_specs(self): if d not in needed: yield d - def _get_environment_specs(self, recurse_dependencies=True): - """Returns the specs of all the packages in an environment. - - If these specs appear under different user_specs, only one copy - is added to the list returned. - """ - specs = [self.specs_by_hash[h] for h in self.all_concretized_orders()] - if recurse_dependencies: - specs.extend( - traverse.traverse_nodes( - specs, root=False, deptype=("link", "run"), key=traverse.by_dag_hash - ) - ) - - return specs - def _concrete_specs_dict(self): concrete_specs = {} for s in traverse.traverse_nodes(self.specs_by_hash.values(), key=traverse.by_dag_hash): @@ -2235,11 +2248,21 @@ def _concrete_specs_dict(self): return concrete_specs def _concrete_roots_dict(self): - hash_spec_list = zip(self.concretized_order, self.concretized_user_specs) - return [{"hash": h, "spec": str(s)} for h, s in hash_spec_list] + if not self.has_groups(): + return [{"hash": x.hash, "spec": str(x.root)} for x in self.concretized_roots] + + return [ + {"hash": x.hash, "spec": str(x.root), "group": x.group} for x in self.concretized_roots + ] + + def has_groups(self) -> bool: + groups = self.manifest.groups() + # True if groups != {DEFAULT_USER_SPEC_GROUP} + return len(groups) != 1 or DEFAULT_USER_SPEC_GROUP not in groups def _to_lockfile_dict(self): """Create a dictionary to store a lockfile for this environment.""" + lockfile_version = CURRENT_LOCKFILE_VERSION if self.has_groups() else 6 concrete_specs = self._concrete_specs_dict() root_specs = self._concrete_roots_dict() @@ -2256,7 +2279,7 @@ def _to_lockfile_dict(self): # metadata about the format "_meta": { "file-type": "spack-lockfile", - "lockfile-version": lockfile_format_version, + "lockfile-version": lockfile_version, "specfile-version": spack.spec.SPECFILE_FORMAT_VERSION, }, # spack version information @@ -2267,8 +2290,8 @@ def _to_lockfile_dict(self): "concrete_specs": concrete_specs, } - if self.included_concrete_envs: - data[included_concrete_name] = self.included_concrete_spec_data + if self.included_concrete_env_root_dirs: + data[lockfile_include_key] = self.included_concrete_spec_data return data @@ -2278,39 +2301,38 @@ def _read_lockfile(self, file_or_json): self._read_lockfile_dict(lockfile_dict) return lockfile_dict - def set_included_concretized_user_specs( + def _set_included_env_roots( self, env_name: str, env_info: Dict[str, Dict[str, Any]], included_json_specs_by_hash: Dict[str, Dict[str, Any]], ) -> Dict[str, Dict[str, Any]]: - """Sets all of the concretized user specs from included environments - to include those from nested included environments. + """Populates included_concretized_roots from included environment data, + including any transitively nested included environments. Args: - env_name: the name (technically the path) of the included environment + env_name: the path of the included environment env_info: included concrete environment data included_json_specs_by_hash: concrete spec data keyed by hash Returns: updated specs_by_hash """ - self.included_concretized_order[env_name] = [] - self.included_concretized_user_specs[env_name] = [] + self.included_concretized_roots[env_name] = [] def add_specs(name, info, specs_by_hash): # Add specs from the environment as well as any of its nested # environments. for root_info in info["roots"]: - self.included_concretized_order[name].append(root_info["hash"]) - self.included_concretized_user_specs[name].append(Spec(root_info["spec"])) + self.included_concretized_roots[name].append( + ConcretizedRootInfo.from_info_dict(root_info) + ) if "concrete_specs" in info: specs_by_hash.update(info["concrete_specs"]) - if included_concrete_name in info: - for included_name, included_info in info[included_concrete_name].items(): - if included_name not in self.included_concretized_order: - self.included_concretized_order[included_name] = [] - self.included_concretized_user_specs[included_name] = [] + if lockfile_include_key in info: + for included_name, included_info in info[lockfile_include_key].items(): + if included_name not in self.included_concretized_roots: + self.included_concretized_roots[included_name] = [] add_specs(included_name, included_info, specs_by_hash) add_specs(env_name, env_info, included_json_specs_by_hash) @@ -2320,21 +2342,18 @@ def _read_lockfile_dict(self, d): """Read a lockfile dictionary into this environment.""" self.specs_by_hash = {} self.included_specs_by_hash = {} - self.included_concretized_user_specs = {} - self.included_concretized_order = {} + self.included_concretized_roots = {} roots = d["roots"] - self.concretized_user_specs = [Spec(r["spec"]) for r in roots] - self.concretized_order = [r["hash"] for r in roots] + self.concretized_roots = [ConcretizedRootInfo.from_info_dict(r) for r in roots] + json_specs_by_hash = d["concrete_specs"] included_json_specs_by_hash = {} - if included_concrete_name in d: - for env_name, env_info in d[included_concrete_name].items(): + if lockfile_include_key in d: + for env_name, env_info in d[lockfile_include_key].items(): included_json_specs_by_hash.update( - self.set_included_concretized_user_specs( - env_name, env_info, included_json_specs_by_hash - ) + self._set_included_env_roots(env_name, env_info, included_json_specs_by_hash) ) current_lockfile_format = d["_meta"]["lockfile-version"] @@ -2345,35 +2364,35 @@ def _read_lockfile_dict(self, d): f"Spack {spack.__version__} cannot read the lockfile '{self.lock_path}', using " f"the v{current_lockfile_format} format." ) - if lockfile_format_version < current_lockfile_format: + if CURRENT_LOCKFILE_VERSION < current_lockfile_format: msg += " You need to use a newer Spack version." raise SpackEnvironmentError(msg) - first_seen, self.concretized_order = self.filter_specs( - reader, json_specs_by_hash, self.concretized_order + concretized_order = [x.hash for x in self.concretized_roots] + first_seen, concretized_order = self._filter_specs( + reader, json_specs_by_hash, concretized_order ) - - for spec_dag_hash in self.concretized_order: + for idx, spec_dag_hash in enumerate(concretized_order): + self.concretized_roots[idx].hash = spec_dag_hash self.specs_by_hash[spec_dag_hash] = first_seen[spec_dag_hash] - if any(self.included_concretized_order.values()): + if any(self.included_concretized_roots.values()): first_seen = {} - for env_name, concretized_order in self.included_concretized_order.items(): - filtered_spec, self.included_concretized_order[env_name] = self.filter_specs( - reader, included_json_specs_by_hash, concretized_order + for env_name, roots in self.included_concretized_roots.items(): + order = [x.hash for x in roots] + filtered_spec, new_order = self._filter_specs( + reader, included_json_specs_by_hash, order ) first_seen.update(filtered_spec) + for idx, spec_dag_hash in enumerate(new_order): + roots[idx].hash = spec_dag_hash - for env_path, spec_hashes in self.included_concretized_order.items(): - self.included_specs_by_hash[env_path] = {} - for spec_dag_hash in spec_hashes: - self.included_specs_by_hash[env_path].update( - {spec_dag_hash: first_seen[spec_dag_hash]} - ) + for env_path, roots in self.included_concretized_roots.items(): + self.included_specs_by_hash[env_path] = {x.hash: first_seen[x.hash] for x in roots} - def filter_specs(self, reader, json_specs_by_hash, order_concretized): - # Track specs by their lockfile key. Currently spack uses the finest + def _filter_specs(self, reader, json_specs_by_hash, order_concretized): + # Track specs by their lockfile key. Currently, spack uses the finest # grained hash as the lockfile key, while older formats used the build # hash or a previous incarnation of the DAG hash (one that did not # include build deps or package hash). @@ -2438,7 +2457,7 @@ def write(self, regenerate: bool = True) -> None: regenerate: regenerate views and run post-write hooks as well as writing if True. """ self.manifest_uptodate_or_warn() - if self.specs_by_hash or self.included_concrete_envs: + if self.specs_by_hash or self.included_concrete_env_root_dirs: self.ensure_env_directory_exists(dot_env=True) self.update_environment_repository() self.manifest.flush() @@ -2455,7 +2474,8 @@ def write(self, regenerate: bool = True) -> None: if regenerate: self.regenerate_views() - self.new_specs.clear() + for x in self.concretized_roots: + x.new = False def update_lockfile(self) -> None: with fs.write_tmp_and_move(self.lock_path, encoding="utf-8") as f: @@ -2473,7 +2493,8 @@ def ensure_env_directory_exists(self, dot_env: bool = False) -> None: def update_environment_repository(self) -> None: """Updates the repository associated with the environment.""" - for spec in traverse.traverse_nodes(self.new_specs): + new_specs = [self.specs_by_hash[x.hash] for x in self.concretized_roots if x.new] + for spec in traverse.traverse_nodes(new_specs): if not spec.concrete: raise ValueError("specs passed to environment.write() must be concrete!") @@ -2534,6 +2555,308 @@ def __exit__(self, exc_type, exc_val, exc_tb): activate(self._previous_active) +def _is_uninstalled(spec): + return not spec.installed or (spec.satisfies("dev_path=*") or spec.satisfies("^dev_path=*")) + + +class ReusableSpecsFactory: + """Creates a list of SpecFilters to generate the reusable specs for the environment""" + + def __init__(self, *, env: Environment, group: str): + self.env = env + self.group = group + + @staticmethod + def _const(specs: List[Spec]) -> Callable[[], List[Spec]]: + """Returns a zero-argument callable that always returns the given list.""" + return lambda: specs + + def __call__( + self, is_usable: Callable[[Spec], bool], configuration: spack.config.Configuration + ) -> List[SpecFilter]: + result = [] + # Specs from group dependencies _must_ be reused, regardless of configuration + dependencies = self.env.manifest.needs(group=self.group) + necessary_specs = [] + for d in dependencies: + necessary_specs.extend([x for _, x in self.env.concretized_specs_by(group=d)]) + + # Specs from groups listed as dependencies + if necessary_specs: + necessary_specs = list( + traverse.traverse_nodes(necessary_specs, deptype=("link", "run")) + ) + result.append( + SpecFilter( + self._const(necessary_specs), include=[], exclude=[], is_usable=is_usable + ) + ) + + # Included environments and _this_ group, instead, are subject to configuration + concretizer_yaml = configuration.get_config("concretizer") + reuse_yaml = concretizer_yaml.get("reuse", False) + + # With no reuse don't account for previously concretized specs in _this_ group + if reuse_yaml is False: + return result + + this_group_specs = [x for _, x in self.env.concretized_specs_by(group=self.group)] + included_specs = [ + x for _, x in self.env.concretized_specs_from_all_included_environments() + ] + additional_specs = list(traverse.traverse_nodes(this_group_specs + included_specs)) + if not isinstance(reuse_yaml, Mapping): + result.append( + SpecFilter( + self._const(additional_specs), include=[], exclude=[], is_usable=is_usable + ) + ) + return result + + # Here we know we have a complex reuse configuration + default_include = reuse_yaml.get("include", []) + default_exclude = reuse_yaml.get("exclude", []) + for source in reuse_yaml.get("from", []): + # We just need to take care of the environment-related parts + if source["type"] != "environment": + continue + + include = source.get("include", default_include) + exclude = source.get("exclude", default_exclude) + if "path" not in source: + result.append( + SpecFilter( + self._const(additional_specs), + include=include, + exclude=exclude, + is_usable=is_usable, + ) + ) + continue + + env_dir = as_env_dir(source["path"]) + if env_dir in self.env.included_concrete_env_root_dirs: + spec_pairs_from_included_envs = [ + x for _, x in self.env.concretized_specs_from_included_environment(env_dir) + ] + included_specs = list(traverse.traverse_nodes(spec_pairs_from_included_envs)) + result.append( + SpecFilter( + self._const(included_specs), + include=include, + exclude=exclude, + is_usable=is_usable, + ) + ) + + return result + + +class EnvironmentConcretizer: + def __init__(self, env: Environment): + self.env = env + + def concretize( + self, *, force: Optional[bool] = None, tests: Union[bool, Sequence[str]] = False + ) -> List[SpecPair]: + if force is None: + force = spack.config.get("concretizer:force") + self._prepare_environment_for_concretization(force=force) + + result = [] + # Sort so that the ordering is deterministic, and "default" specs are first + for current_group in self._order_groups(): + with self.env.config_override_for_group(group=current_group): + partial_result = self._concretize_single_group(group=current_group, tests=tests) + result.extend(partial_result) + + # Unify the specs objects, so we get correct references to all parents + if result: + self.env.unify_specs() + return result + + def _concretize_single_group( + self, *, group: str, tests: Union[bool, Sequence[str]] + ) -> List[SpecPair]: + # Exit early if the set of concretized specs is the set of user specs + new_user_specs, kept_user_specs = self._partition_user_specs(group=group) + if not new_user_specs: + return [] + + # Pick the right concretization strategy + if group != DEFAULT_USER_SPEC_GROUP: + tty.msg(f"Concretizing the '{group}' group of specs") + unify = spack.config.CONFIG.get_config("concretizer").get("unify", False) + factory = ReusableSpecsFactory(env=self.env, group=group) + if unify == "when_possible": + partial_result = self._concretize_together_where_possible( + new_user_specs, kept_user_specs, tests=tests, group=group, factory=factory + ) + + elif unify is True: + partial_result = self._concretize_together( + new_user_specs, kept_user_specs, tests=tests, group=group, factory=factory + ) + + elif unify is False: + partial_result = self._concretize_separately( + new_user_specs, kept_user_specs, tests=tests, group=group, factory=factory + ) + else: + raise SpackEnvironmentError(f"concretization strategy not implemented [{unify}]") + + return partial_result + + def _prepare_environment_for_concretization(self, *, force: bool): + """Reset the environment concrete state and ensure consistency with user specs.""" + if force: + self.env.clear_concretized_specs() + else: + self.env.sync_concretized_specs() + + # If a combined env, check updated spec is in the linked envs + if self.env.included_concrete_env_root_dirs: + self.env.include_concrete_envs() + + def _partition_user_specs( + self, *, group: str + ) -> Tuple[List[spack.spec.Spec], List[spack.spec.Spec]]: + """Splits the users specs in the list of the ones to be computed, and the list of + the ones to retain. + """ + concretized_user_specs = {x.root for x in self.env.concretized_roots if x.group == group} + kept_user_specs, new_user_specs = stable_partition( + self.env.user_specs_by(group=group), lambda x: x in concretized_user_specs + ) + kept_user_specs += self.env.included_user_specs + return new_user_specs, kept_user_specs + + def _order_groups(self) -> List[str]: + done, result = {DEFAULT_USER_SPEC_GROUP}, [DEFAULT_USER_SPEC_GROUP] + all_groups = self.env.manifest.groups() + remaining = all_groups - {DEFAULT_USER_SPEC_GROUP} + + # Validate upfront that all 'needs' references point to defined groups + for group in remaining: + for dep in self.env.manifest.needs(group=group): + if dep not in all_groups: + raise SpackEnvironmentConfigError( + f"group '{group}' needs '{dep}', but '{dep}' is not a defined group", + self.env.manifest.manifest_file, + ) + + while remaining: + # Check we have groups that are "ready" + ready = [] + for current in remaining: + deps = self.env.manifest.needs(group=current) + if all(d in done for d in deps): + ready.append(current) + + # Check we can progress - if nothing is ready, there is a cycle + if not ready: + raise SpackEnvironmentConfigError( + f"cyclic dependency detected among groups: {', '.join(sorted(remaining))}", + self.env.manifest.manifest_file, + ) + + result.extend(ready) + done.update(ready) + remaining.difference_update(ready) + return result + + def _user_spec_pairs( + self, user_specs_to_compute: List[Spec], user_specs_to_keep: List[Spec] + ) -> List[SpecPair]: + specs_to_concretize = [(s, None) for s in user_specs_to_compute] + [ + (abstract, concrete) + for abstract, concrete in self.env.concretized_specs() + if abstract in user_specs_to_keep + ] + return specs_to_concretize + + def _concretize_together_where_possible( + self, + to_compute: List[Spec], + to_keep: List[Spec], + *, + group: Optional[str] = None, + tests: Union[bool, Sequence] = False, + factory: ReusableSpecsFactory, + ) -> List[SpecPair]: + import spack.concretize + + specs_to_concretize = self._user_spec_pairs(to_compute, to_keep) + result = spack.concretize.concretize_together_when_possible( + specs_to_concretize, tests=tests, factory=factory + ) + result = [x for x in result if x[0] in to_compute] + for abstract, concrete in result: + self.env.add_concrete_spec(abstract, concrete, new=True, group=group) + + return result + + def _concretize_together( + self, + to_compute: List[Spec], + to_keep: List[Spec], + *, + group: Optional[str] = None, + tests: Union[bool, Sequence] = False, + factory: ReusableSpecsFactory, + ) -> List[SpecPair]: + import spack.concretize + + to_concretize = self._user_spec_pairs(to_compute, to_keep) + try: + concrete_pairs = spack.concretize.concretize_together( + to_concretize, tests=tests, factory=factory + ) + except spack.error.UnsatisfiableSpecError as e: + # "Enhance" the error message for multiple root specs, suggest a less strict + # form of concretization. + if len(self.env.user_specs_by(group=group)) > 1: + e.message += ". " + if to_keep: + e.message += ( + "Couldn't concretize without changing the existing environment. " + "If you are ok with changing it, try `spack concretize --force`. " + ) + e.message += ( + "You could consider setting `concretizer:unify` to `when_possible` " + "or `false` to allow multiple versions of some packages." + ) + raise + + # Return the portion of the return value that is new + result = concrete_pairs[: len(to_compute)] + for abstract, concrete in result: + self.env.add_concrete_spec(abstract, concrete, new=True, group=group) + return result + + def _concretize_separately( + self, + to_compute: List[Spec], + to_keep: List[Spec], + *, + group: Optional[str] = None, + tests: Union[bool, Sequence] = False, + factory: ReusableSpecsFactory, + ) -> List[SpecPair]: + """Concretization strategy that concretizes separately one user spec after the other""" + import spack.concretize + + to_concretize = [(x, None) for x in to_compute] + concrete_pairs = spack.concretize.concretize_separately( + to_concretize, tests=tests, factory=factory + ) + + for abstract, concrete in concrete_pairs: + self.env.add_concrete_spec(abstract, concrete, new=True, group=group) + + return concrete_pairs + + def yaml_equivalent(first, second) -> bool: """Returns whether two spack yaml items are equivalent, including overrides""" # YAML has timestamps and dates, but we don't use them yet in schemas @@ -2783,7 +3106,7 @@ def _ensure_env_dir(): return # TODO: make this recursive - includes = manifest[TOP_LEVEL_KEY].get("include", []) + includes = manifest[TOP_LEVEL_KEY].get(manifest_include_name, []) paths = spack.config.paths_from_includes(includes) for path in paths: if os.path.isabs(path): @@ -2833,13 +3156,22 @@ def from_lockfile(manifest_dir: Union[pathlib.Path, str]) -> "EnvironmentManifes lockfile = manifest_dir / lockfile_name with lockfile.open("r", encoding="utf-8") as f: data = sjson.load(f) - user_specs = data["roots"] + roots = data["roots"] + + user_specs_by_group: Dict[str, List[str]] = {} + for item in roots: + # "group" is not there for Lockfile v6 and lower + group = item.get("group", DEFAULT_USER_SPEC_GROUP) + user_specs_by_group.setdefault(group, []).append(item["spec"]) default_content = manifest_dir / manifest_name default_content.write_text(default_manifest_yaml()) manifest = EnvironmentManifestFile(manifest_dir) - for item in user_specs: - manifest.add_user_spec(item["spec"]) + + for group, specs in user_specs_by_group.items(): + for spec in specs: + manifest.add_user_spec(spec, group=group) + manifest.flush() return manifest @@ -2861,8 +3193,51 @@ def __init__(self, manifest_dir: Union[pathlib.Path, str], name: Optional[str] = with self.manifest_file.open(encoding="utf-8") as f: self.yaml_content = _read_yaml(f) + # Maps groups to their dependencies + self._groups: Dict[str, Tuple[str, ...]] = {DEFAULT_USER_SPEC_GROUP: tuple()} + # Raw YAML definitions of the user specs for each group + self._user_specs: Dict[str, List] = {DEFAULT_USER_SPEC_GROUP: []} + # Configuration overrides for each group + self._config_override: Dict[str, Any] = {DEFAULT_USER_SPEC_GROUP: None} + # Whether specs in each group are marked explicit + self._explicit: Dict[str, bool] = {DEFAULT_USER_SPEC_GROUP: True} + self._init_user_specs() + self.changed = False + def _init_user_specs(self): + specs_yaml = self.configuration.get(USER_SPECS_KEY, []) + for item in specs_yaml: + if isinstance(item, str): + self._user_specs[DEFAULT_USER_SPEC_GROUP].append(item) + elif isinstance(item, dict): + group = item.get("group", DEFAULT_USER_SPEC_GROUP) + + # Error if a group is defined more than once + if group != DEFAULT_USER_SPEC_GROUP and group in self._groups: + raise SpackEnvironmentConfigError( + f"group '{group}' defined more than once", self.manifest_file + ) + + # Add an entry for the user specs and store group dependencies + if group not in self._user_specs: + self._user_specs[group] = [] + self._groups[group] = tuple(item.get("needs", ())) + self._config_override[group] = item.get("override", None) + self._explicit[group] = item.get("explicit", True) + + if "matrix" in item: + # Short form if the group is composed of only one matrix + self._user_specs[group].append({"matrix": item["matrix"]}) + elif "specs" in item: + self._user_specs[group].extend(item["specs"]) + + def _clear_user_specs(self) -> None: + self._user_specs = {DEFAULT_USER_SPEC_GROUP: []} + self._groups = {DEFAULT_USER_SPEC_GROUP: tuple()} + self._config_override = {DEFAULT_USER_SPEC_GROUP: None} + self._explicit = {DEFAULT_USER_SPEC_GROUP: True} + def _all_matches(self, user_spec: str) -> List[str]: """Maps the input string to the first equivalent user spec in the manifest, and returns it. @@ -2883,17 +3258,85 @@ def _all_matches(self, user_spec: str) -> List[str]: return result - def add_user_spec(self, user_spec: str) -> None: - """Appends the user spec passed as input to the list of root specs. + def user_specs(self, *, group: Optional[str] = None) -> List: + group = self._ensure_group_exists(group) + return self._user_specs[group] + + def config_override( + self, *, group: Optional[str] = None + ) -> Optional[spack.config.InternalConfigScope]: + group = self._ensure_group_exists(group) + data = self._config_override[group] + if data is None: + return None + return spack.config.InternalConfigScope(f"env:groups:{group}", data) + + def groups(self) -> KeysView: + """Returns the list of groups defined in the manifest""" + return self._groups.keys() + + def needs(self, *, group: Optional[str] = None) -> Tuple[str, ...]: + """Returns the dependencies of a group of user specs.""" + group = self._ensure_group_exists(group) + return self._groups[group] + + def is_explicit(self, *, group: Optional[str] = None) -> bool: + """Returns whether specs in a group are marked explicit. + + When False, specs in the group are installed as implicit dependencies + and are eligible for garbage collection once no other spec depends on them. + """ + group = self._ensure_group_exists(group) + return self._explicit[group] + + def _ensure_group_exists(self, group: Optional[str]) -> str: + group = DEFAULT_USER_SPEC_GROUP if group is None else group + if group not in self._groups: + raise ValueError(f"user specs group '{group}' not found in {self.manifest_file}") + return group + + def add_user_spec(self, user_spec: str, *, group: Optional[str] = None) -> None: + """Appends the user spec passed as input to the list of root specs for the given group. Args: user_spec: user spec to be appended + group: group where the spec should be added. If None, the default group is used. """ - self.configuration.setdefault("specs", []).append(user_spec) + group = group or DEFAULT_USER_SPEC_GROUP + + if group == DEFAULT_USER_SPEC_GROUP: + # Append to top-most specs: attribute + specs_yaml = self.configuration.setdefault("specs", []) + specs_yaml.append(user_spec) + else: + # Append to specs: attribute within a group + group_in_yaml = self._get_group(group) + group_in_yaml.setdefault("specs", []).append(user_spec) + + self._user_specs[group].append(user_spec) self.changed = True + def _get_group(self, group: str) -> Dict: + """Find or create the group entry in the manifest""" + specs_yaml = self.configuration.setdefault("specs", []) + group_entry = None + for item in specs_yaml: + if isinstance(item, dict) and item.get("group") == group: + group_entry = item + break + + if group_entry is None: + group_entry = {"group": group, "specs": []} + specs_yaml.append(group_entry) + self._groups[group] = tuple() + self._config_override[group] = None + self._user_specs[group] = [] + self._explicit[group] = True + + return group_entry + def remove_user_spec(self, user_spec: str) -> None: - """Removes the user spec passed as input from the list of root specs + """Removes the user spec passed as input from the default list of root specs Args: user_spec: user spec to be removed @@ -2904,6 +3347,7 @@ def remove_user_spec(self, user_spec: str) -> None: try: for key in self._all_matches(user_spec): self.configuration["specs"].remove(key) + self._user_specs[DEFAULT_USER_SPEC_GROUP].remove(key) except ValueError as e: msg = f"cannot remove {user_spec} from {self}, no such spec exists" raise SpackEnvironmentError(msg) from e @@ -2912,6 +3356,7 @@ def remove_user_spec(self, user_spec: str) -> None: def clear(self) -> None: """Clear all user specs from the list of root specs""" self.configuration["specs"] = [] + self._clear_user_specs() self.changed = True def override_user_spec(self, user_spec: str, idx: int) -> None: @@ -2926,6 +3371,8 @@ def override_user_spec(self, user_spec: str, idx: int) -> None: """ try: self.configuration["specs"][idx] = user_spec + self._clear_user_specs() + self._init_user_specs() except ValueError as e: msg = f"cannot override {user_spec} from {self}" raise SpackEnvironmentError(msg) from e @@ -2937,11 +3384,7 @@ def set_include_concrete(self, include_concrete: List[str]) -> None: Args: include_concrete: list of already existing concrete environments to include """ - self.configuration[included_concrete_name] = [] - - for env_path in include_concrete: - self.configuration[included_concrete_name].append(env_path) - + self.configuration[lockfile_include_key] = list(include_concrete) self.changed = True def add_definition(self, user_spec: str, list_name: str) -> None: @@ -3157,8 +3600,7 @@ class SpackEnvironmentConfigError(SpackEnvironmentError): """Class for Spack environment-specific configuration errors.""" def __init__(self, msg, filename): - self.filename = filename - super().__init__(msg) + super().__init__(f"{msg} in {filename}") class SpackEnvironmentDevelopError(SpackEnvironmentError): diff --git a/lib/spack/spack/environment/list.py b/lib/spack/spack/environment/list.py index 9bb258fa777ce3..a2097f8e80493c 100644 --- a/lib/spack/spack/environment/list.py +++ b/lib/spack/spack/environment/list.py @@ -9,10 +9,13 @@ import spack.variant from spack.error import SpackError from spack.spec import Spec +from spack.spec_parser import expand_toolchains class SpecList: - def __init__(self, *, name: str = "specs", yaml_list=None, expanded_list=None): + def __init__( + self, *, name: str = "specs", yaml_list=None, expanded_list=None, toolchains=None + ): self.name = name self.yaml_list = yaml_list[:] if yaml_list is not None else [] # Expansions can be expensive to compute and difficult to keep updated @@ -20,6 +23,7 @@ def __init__(self, *, name: str = "specs", yaml_list=None, expanded_list=None): self.specs_as_yaml_list = expanded_list or [] self._constraints = None self._specs: Optional[List[Spec]] = None + self._toolchains = toolchains @property def is_matrix(self): @@ -51,6 +55,8 @@ def specs(self) -> List[Spec]: spec = constraint_list[0].copy() for const in constraint_list[1:]: spec.constrain(const) + if self._toolchains: + expand_toolchains(spec, self._toolchains) specs.append(spec) self._specs = specs @@ -178,8 +184,9 @@ class Definition(NamedTuple): class SpecListParser: """Parse definitions and user specs from data in environments""" - def __init__(self): + def __init__(self, *, toolchains=None): self.definitions: Dict[str, SpecList] = {} + self._toolchains = toolchains def parse_definitions(self, *, data: List[Dict[str, Any]]) -> Dict[str, SpecList]: definitions_from_yaml: Dict[str, List[Definition]] = {} @@ -225,7 +232,12 @@ def _speclist_from_definitions(self, name, definitions) -> SpecList: continue combined_yaml_list.extend(def_part.yaml_list) expanded_list = self._expand_yaml_list(combined_yaml_list) - return SpecList(name=name, yaml_list=combined_yaml_list, expanded_list=expanded_list) + return SpecList( + name=name, + yaml_list=combined_yaml_list, + expanded_list=expanded_list, + toolchains=self._toolchains, + ) def _expand_yaml_list(self, raw_yaml_list): result = [] diff --git a/lib/spack/spack/environment/shell.py b/lib/spack/spack/environment/shell.py index 957d52d829ee12..f639c25cc90e81 100644 --- a/lib/spack/spack/environment/shell.py +++ b/lib/spack/spack/environment/shell.py @@ -194,7 +194,7 @@ def activate( "Environment view is broken due to a missing package or repo.\n", " To activate without views enabled, activate with:\n", " spack env activate -V {0}\n".format(env.name), - " To remove it and resolve the issue, " "force concretize with the command:\n", + " To remove it and resolve the issue, force concretize with the command:\n", " spack -e {0} concretize --force".format(env.name), ) diff --git a/lib/spack/spack/error.py b/lib/spack/spack/error.py index de90ed053dfdf7..1ab2f95076d545 100644 --- a/lib/spack/spack/error.py +++ b/lib/spack/spack/error.py @@ -116,6 +116,10 @@ class SpecError(SpackError): """Superclass for all errors that occur while constructing specs.""" +class InvalidVirtualOnEdgeError(SpecError): + """Raised when an edge requires a virtual that does not exist in the repository.""" + + class UnsatisfiableSpecError(SpecError): """ Raised when a spec conflicts with package constraints. diff --git a/lib/spack/spack/extensions.py b/lib/spack/spack/extensions.py index 19183ab17ca8d7..895b136b29eba6 100644 --- a/lib/spack/spack/extensions.py +++ b/lib/spack/spack/extensions.py @@ -4,6 +4,7 @@ """Service functions and classes to implement the hooks for Spack's command extensions. """ + import glob import importlib import os diff --git a/lib/spack/spack/externals.py b/lib/spack/spack/externals.py index aa1c66d9a15050..a86d6e737071e5 100644 --- a/lib/spack/spack/externals.py +++ b/lib/spack/spack/externals.py @@ -14,6 +14,7 @@ The helper function ``extract_dicts_from_configuration`` is used to transform the configuration into the intermediate representation. """ + import re import uuid import warnings @@ -115,6 +116,15 @@ def complete_variants_and_architecture(node: spack.spec.Spec) -> None: if name not in node.variants: # Cannot use Spec.constrain, because we lose information on the variant type node.variants[name] = vdef.make_default() + elif ( + node.variants[name].type != vdef.variant_type + and len(node.variants[name].values) == 1 + ): + # Spec parsing defaults to MULTI for non-boolean variants. Correct the type + # using the package definition, preserving the user-specified value. + existing = node.variants[name] + corrected = vdef.make_variant(*existing.values) + node.variants.substitute(corrected) changed = True diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 1d2d5b4d1b2297..d07ccc21b4babb 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -25,12 +25,14 @@ ``archive()`` Archive a source directory, e.g. for creating a mirror. """ + import copy import functools import hashlib import http.client import os import re +import secrets import shutil import sys import time @@ -427,43 +429,53 @@ def _check_headers(self, headers): tty.warn(msg) @_needs_stage - def _fetch_urllib(self, url, chunk_size=65536): + def _fetch_urllib(self, url, chunk_size=65536, retries=5): + """Fetch a URL using urllib, with retries on transient errors and progress reporting.""" save_file = self.stage.save_filename + part_file = save_file + ".part" request = urllib.request.Request( url, headers={"User-Agent": web_util.SPACK_USER_AGENT, "Accept": "*/*"} ) - if os.path.lexists(save_file): - os.remove(save_file) - - try: - response = web_util.urlopen(request) - tty.verbose(f"Fetching {url}") - progress = FetchProgress.from_headers(response.headers, enabled=sys.stdout.isatty()) - with open(save_file, "wb") as f: - while True: - chunk = response.read(chunk_size) - if not chunk: - break - f.write(chunk) - progress.advance(len(chunk)) - progress.print(final=True) - except OSError as e: - # clean up archive on failure. - if self.archive_file: - os.remove(self.archive_file) - if os.path.lexists(save_file): - os.remove(save_file) - raise FailedDownloadError(e) from e + response_headers_str = None + for attempt in range(retries): + try: + with web_util.urlopen(request) as response: + tty.verbose(f"Fetching {url}") + progress = FetchProgress.from_headers( + response.headers, enabled=sys.stdout.isatty() + ) + with open(part_file, "wb") as f: + while True: + chunk = response.read(chunk_size) + if not chunk: + break + f.write(chunk) + progress.advance(len(chunk)) + progress.print(final=True) + # Capture metadata before context manager closes the connection + if isinstance(response, http.client.HTTPResponse): + self._effective_url = response.geturl() + response_headers_str = str(response.headers) + os.replace(part_file, save_file) + break # success: exit retry loop + except Exception as e: + # clean up archive on failure. + if self.archive_file: + os.remove(self.archive_file) + if os.path.lexists(part_file): + os.remove(part_file) + # Raise if this was the last attempt, or if the error was not transient. + if (attempt + 1 == retries) or not web_util.is_transient_error(e): + raise FailedDownloadError(e) from e + tty.debug(f"Retrying fetch (attempt {attempt + 1}): {e}") + time.sleep(2**attempt) # Save the redirected URL for error messages. Sometimes we're redirected to an arbitrary # mirror that is broken, leading to spurious download failures. In that case it's helpful # for users to know which URL was actually fetched. - if isinstance(response, http.client.HTTPResponse): - self._effective_url = response.geturl() - - self._check_headers(str(response.headers)) + self._check_headers(response_headers_str) @_needs_stage def _fetch_curl(self, url, config_args=[]): @@ -961,12 +973,19 @@ def _clone_src(self) -> None: kwargs = {"debug": spack.config.get("config:debug"), "git_exe": self.git, "dest": name} + # TODO(psakievich) The use of the minimal clone need clearer justification via package API + # or something. There is a trade space of storage minimization vs available git information + # that grows to non-trivial proportions for larger projects + minimal_clone = self.commit and name and not self.get_full_repo + with temp_cwd(ignore_cleanup_errors=True): - if self.commit and name: + if minimal_clone: try: spack.util.git.git_init_fetch(self.url, self.commit, depth, **kwargs) except spack.util.executable.ProcessError: - spack.util.git.git_clone(self.url, fetch_ref, True, depth, **kwargs) + spack.util.git.git_clone( + self.url, fetch_ref, self.get_full_repo, depth, **kwargs + ) else: spack.util.git.git_clone(self.url, fetch_ref, self.get_full_repo, depth, **kwargs) repo_name = get_single_file(".") @@ -1598,7 +1617,8 @@ def _for_package_version(pkg, version=None): commit = commit_var.value if commit_var else None tag = None if isinstance(version, spack.version.GitVersion) or commit: - if not hasattr(pkg, "git"): + git_url = pkg.version_or_package_attr("git", version) + if not git_url: raise spack.error.FetchError( f"Cannot fetch git version for {pkg.name}. Package has no 'git' attribute" ) @@ -1630,9 +1650,10 @@ def _for_package_version(pkg, version=None): tag = version_meta_data.get("tag") or version_meta_data.get("branch") kwargs = {"commit": commit, "tag": tag, "no_cache": bool(not commit)} - kwargs["git"] = pkg.version_or_package_attr("git", version) + kwargs["git"] = git_url kwargs["submodules"] = pkg.version_or_package_attr("submodules", version, False) kwargs["git_sparse_paths"] = pkg.version_or_package_attr("git_sparse_paths", version, None) + kwargs["get_full_repo"] = pkg.version_or_package_attr("get_full_repo", version, False) # if the ref_version is a known version from the package, use that version's # attributes @@ -1740,10 +1761,29 @@ def from_list_url(pkg): tty.msg("Could not determine url from list_url.") -class FsCache: +class FsCacheBase: def __init__(self, root): self.root = os.path.abspath(root) + def store(self, fetcher, relative_dest): + dst = os.path.join(self.root, relative_dest) + mkdirp(os.path.dirname(dst)) + tmp = os.path.join( + os.path.dirname(dst), ".tmp." + secrets.token_hex(6) + "." + os.path.basename(dst) + ) + open(tmp, "xb").close() + try: + fetcher.archive(tmp) + os.replace(tmp, dst) + except BaseException: + try: + os.unlink(tmp) + except OSError: + pass + raise + + +class FsCache(FsCacheBase): def store(self, fetcher, relative_dest): # skip fetchers that aren't cachable if not fetcher.cachable: @@ -1753,9 +1793,7 @@ def store(self, fetcher, relative_dest): if isinstance(fetcher, CacheURLFetchStrategy): return - dst = os.path.join(self.root, relative_dest) - mkdirp(os.path.dirname(dst)) - fetcher.archive(dst) + super().store(fetcher, relative_dest) def fetcher(self, target_path: str, digest: Optional[str], **kwargs) -> CacheURLFetchStrategy: path = os.path.join(self.root, target_path) diff --git a/lib/spack/spack/filesystem_view.py b/lib/spack/spack/filesystem_view.py index d331f65d79b2ca..c5b9674abfa5a3 100644 --- a/lib/spack/spack/filesystem_view.py +++ b/lib/spack/spack/filesystem_view.py @@ -3,7 +3,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import functools as ft -import itertools import os import re import shutil @@ -39,8 +38,8 @@ DestinationMergeVisitor, LinkTree, MergeConflictSummary, + MultiPrefixMerger, SingleMergeConflictError, - SourceMergeVisitor, ) from spack.llnl.util.tty.color import colorize @@ -104,9 +103,11 @@ def view_copy( #: Type alias for link types LinkType = Literal["hardlink", "hard", "copy", "relocate", "add", "symlink", "soft"] +CanonicalLinkType = Literal["hardlink", "copy", "symlink"] + #: supported string values for `link_type` in an env, mapped to canonical values -_LINK_TYPES = { +_LINK_TYPES: Dict[LinkType, CanonicalLinkType] = { "hardlink": "hardlink", "hard": "hardlink", "copy": "copy", @@ -119,7 +120,7 @@ def view_copy( _VALID_LINK_TYPES = sorted(set(_LINK_TYPES.values())) -def canonicalize_link_type(link_type: LinkType) -> str: +def canonicalize_link_type(link_type: LinkType) -> CanonicalLinkType: """Return canonical""" canonical = _LINK_TYPES.get(link_type) if not canonical: @@ -163,6 +164,7 @@ def __init__( ignore_conflicts: bool = False, verbose: bool = False, link_type: LinkType = "symlink", + link_dirs: bool = False, ): """ Initialize a filesystem view under the given ``root`` directory with @@ -180,6 +182,7 @@ def __init__( # Setup link function to include view self.link_type = link_type self._link = function_for_link_type(link_type) + self.link_dirs = link_dirs and link_type == "symlink" def link(self, src: str, dst: str, spec: Optional[spack.spec.Spec] = None) -> None: self._link(src, dst, self, spec) @@ -189,7 +192,7 @@ def add_specs(self, *specs: spack.spec.Spec, **kwargs) -> None: Add given specs to view. Should accept ``with_dependencies`` as keyword argument (default - True) to indicate wether or not dependencies should be activated as + True) to indicate whether or not dependencies should be activated as well. Should except an ``exclude`` keyword argument containing a list of @@ -216,11 +219,11 @@ def remove_specs(self, *specs: spack.spec.Spec, **kwargs) -> None: Removes given specs from view. Should accept ``with_dependencies`` as keyword argument (default - True) to indicate wether or not dependencies should be deactivated + True) to indicate whether or not dependencies should be deactivated as well. Should accept ``with_dependents`` as keyword argument (default True) - to indicate wether or not dependents on the deactivated specs + to indicate whether or not dependents on the deactivated specs should be removed as well. Should except an ``exclude`` keyword argument containing a list of @@ -268,7 +271,7 @@ def print_status(self, *specs: spack.spec.Spec, **kwargs) -> None: * ..they are active in the view. * ..they are active but the activated version differs. - * ..they are not activte in the view. + * ..they are not active in the view. Takes ``with_dependencies`` keyword argument so that the status of dependencies is printed as well. @@ -712,13 +715,16 @@ def skip_list(file): # Determine if the root is on a case-insensitive filesystem normalize_paths = is_folder_on_case_insensitive_filesystem(self._root) - visitor = SourceMergeVisitor(ignore=skip_list, normalize_paths=normalize_paths) - - # Gather all the directories to be made and files to be linked - for spec in specs: - src_prefix = spec.package.view_source() - visitor.set_projection(self.get_relative_projection_for_spec(spec)) - visit_directory_tree(src_prefix, visitor) + sources = [ + (spec.package.view_source(), self.get_relative_projection_for_spec(spec)) + for spec in specs + ] + visitor = MultiPrefixMerger( + sources, + ignore=skip_list, + normalize_paths=normalize_paths, + dir_symlink_optimization=self.link_dirs, + ) # Check for conflicts in destination dir. visit_directory_tree(self._root, DestinationMergeVisitor(visitor)) @@ -752,21 +758,17 @@ def skip_list(file): # Finally create the metadata dirs. self.link_metadata(specs) - def _source_merge_visitor_to_merge_map(self, visitor: SourceMergeVisitor): + def _source_merge_visitor_to_merge_map(self, visitor: MultiPrefixMerger): # For compatibility with add_files_to_view, we have to create a # merge_map of the form join(src_root, src_rel) => join(dst_root, dst_rel), # but our visitor.files format is dst_rel => (src_root, src_rel). - # We exploit that visitor.files is an ordered dict, and files per source - # prefix are contiguous. - source_root = lambda item: item[1][0] - per_source = itertools.groupby(visitor.files.items(), key=source_root) - return { - src_root: { - os.path.join(src_root, src_rel): os.path.join(self._root, dst_rel) - for dst_rel, (_, src_rel) in group - } - for src_root, group in per_source - } + merge_map: Dict[str, Dict[str, str]] = {} + for dst_rel, (src_root, src_rel) in visitor.files.items(): + per_source = merge_map.get(src_root) + if per_source is None: + per_source = merge_map[src_root] = {} + per_source[os.path.join(src_root, src_rel)] = os.path.join(self._root, dst_rel) + return merge_map def relative_metadata_dir_for_spec(self, spec): return os.path.join( @@ -776,15 +778,14 @@ def relative_metadata_dir_for_spec(self, spec): ) def link_metadata(self, specs): - metadata_visitor = SourceMergeVisitor() - - for spec in specs: - src_prefix = os.path.join( - spec.package.view_source(), spack.store.STORE.layout.metadata_dir + prefix_and_projection = [ + ( + os.path.join(spec.package.view_source(), spack.store.STORE.layout.metadata_dir), + self.relative_metadata_dir_for_spec(spec), ) - proj = self.relative_metadata_dir_for_spec(spec) - metadata_visitor.set_projection(proj) - visit_directory_tree(src_prefix, metadata_visitor) + for spec in specs + ] + metadata_visitor = MultiPrefixMerger(prefix_and_projection) # Check for conflicts in destination dir. visit_directory_tree(self._root, DestinationMergeVisitor(metadata_visitor)) diff --git a/lib/spack/spack/graph.py b/lib/spack/spack/graph.py index 37f3962a0fbdf8..3835e2c9ad7b47 100644 --- a/lib/spack/spack/graph.py +++ b/lib/spack/spack/graph.py @@ -37,6 +37,7 @@ :func:`graph_dot` will output a graph of a spec (or multiple specs) in dot format. """ + import enum import sys from typing import List, Optional, Set, TextIO, Tuple @@ -522,11 +523,11 @@ def edge_entry(self, edge): colormap = {"build": "dodgerblue", "link": "crimson", "run": "goldenrod"} label = "" if edge.virtuals: - label = f" xlabel=\"virtuals={','.join(edge.virtuals)}\"" + label = f' xlabel="virtuals={",".join(edge.virtuals)}"' return ( edge.parent.dag_hash(), edge.spec.dag_hash(), - f"[color=\"{':'.join(colormap[x] for x in dt.flag_to_tuple(edge.depflag))}\"" + f'[color="{":".join(colormap[x] for x in dt.flag_to_tuple(edge.depflag))}"' + label + "]", ) diff --git a/lib/spack/spack/hooks/__init__.py b/lib/spack/spack/hooks/__init__.py index 02f597c4878fb2..436d04dd3bbfef 100644 --- a/lib/spack/spack/hooks/__init__.py +++ b/lib/spack/spack/hooks/__init__.py @@ -19,6 +19,7 @@ systems (e.g. modules, lmod, etc.) or to add other custom features. """ + import importlib import types from typing import List, Optional diff --git a/lib/spack/spack/hooks/licensing.py b/lib/spack/spack/hooks/licensing.py index 4fb35560afbe51..4411ce28990fdb 100644 --- a/lib/spack/spack/hooks/licensing.py +++ b/lib/spack/spack/hooks/licensing.py @@ -91,9 +91,7 @@ def write_license_file(pkg, license_path): file UNCHANGED. The system may be configured if: - A license file is installed in a default location. -""".format( - pkg.name - ) +""".format(pkg.name) if envvars: txt += """\ @@ -101,9 +99,7 @@ def write_license_file(pkg, license_path): a module file: {0} -""".format( - envvars - ) +""".format(envvars) txt += """\ * Otherwise, depending on the license you have, enter AT THE BEGINNING of @@ -116,18 +112,14 @@ def write_license_file(pkg, license_path): this Spack-global file (relative to the installation prefix). {0} -""".format( - linktargets - ) +""".format(linktargets) if url: txt += """\ * For further information on licensing, see: {0} -""".format( - url - ) +""".format(url) txt += """\ Recap: diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py index 0ba85100e0c760..39b754fee60cde 100644 --- a/lib/spack/spack/hooks/sbang.py +++ b/lib/spack/spack/hooks/sbang.py @@ -2,7 +2,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import filecmp import os import re import shutil @@ -11,13 +10,8 @@ import tempfile import spack.error -import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty -import spack.package_prefs -import spack.paths -import spack.spec import spack.store -from spack.util.socket import _gethostname #: OS-imposed character limit for shebang line: 127 for Linux; 511 for Mac. #: Different Linux distributions have different limits, but 127 is the @@ -26,11 +20,18 @@ system_shebang_limit = 511 else: system_shebang_limit = 127 - -#: Groupdb does not exist on Windows, prevent imports -#: on supported systems -if sys.platform != "win32": - import grp + try: + # searching for line '#define BINPRM_BUF_SIZE 256' in /usr/include/linux/binfmts.h + # the nbr-1 is the sbang limit on the linux platform + sbang_limit_re = re.compile("#define BINPRM_BUF_SIZE ([0-9]+)") + with open("/usr/include/linux/binfmts.h", "r", encoding="utf-8") as f: + for line in f: + m = sbang_limit_re.match(line) + if m: + system_shebang_limit = int(m.group(1)) - 1 + except Exception: + # ignore any error a sane default is set already + pass #: Spack itself also limits the shebang line to at most 4KB, which should be plenty. spack_shebang_limit = 4096 @@ -177,50 +178,6 @@ def filter_shebangs_in_directory(directory, filenames=None): tty.debug("Patched overlong shebang in %s" % path) -def install_sbang(): - """Ensure that ``sbang`` is installed in the root of Spack's install_tree. - - This is the shortest known publicly accessible path, and installing - ``sbang`` here ensures that users can access the script and that - ``sbang`` itself is in a short path. - """ - # copy in a new version of sbang if it differs from what's in spack - sbang_path = sbang_install_path() - if os.path.exists(sbang_path) and filecmp.cmp(spack.paths.sbang_script, sbang_path): - return - - # make $install_tree/bin - sbang_bin_dir = os.path.dirname(sbang_path) - fs.mkdirp(sbang_bin_dir) - - # get permissions for bin dir from configuration files - group_name = spack.package_prefs.get_package_group(spack.spec.Spec("all")) - config_mode = spack.package_prefs.get_package_dir_permissions(spack.spec.Spec("all")) - - if group_name: - os.chmod(sbang_bin_dir, config_mode) # Use package directory permissions - else: - fs.set_install_permissions(sbang_bin_dir) - - # set group on sbang_bin_dir if not already set (only if set in configuration) - # TODO: after we drop python2 support, use shutil.chown to avoid gid lookups that - # can fail for remote groups - if group_name and os.stat(sbang_bin_dir).st_gid != grp.getgrnam(group_name).gr_gid: - os.chown(sbang_bin_dir, os.stat(sbang_bin_dir).st_uid, grp.getgrnam(group_name).gr_gid) - - # copy over the fresh copy of `sbang` - sbang_tmp_path = os.path.join(sbang_bin_dir, f".sbang.{_gethostname()}.{os.getpid()}.tmp") - shutil.copy(spack.paths.sbang_script, sbang_tmp_path) - - # set permissions on `sbang` (including group if set in configuration) - os.chmod(sbang_tmp_path, config_mode) - if group_name: - os.chown(sbang_tmp_path, os.stat(sbang_tmp_path).st_uid, grp.getgrnam(group_name).gr_gid) - - # Finally, move the new `sbang` into place atomically - os.rename(sbang_tmp_path, sbang_path) - - def post_install(spec, explicit=None): """This hook edits scripts so that they call /bin/bash $spack_prefix/bin/sbang instead of something longer than the @@ -232,8 +189,6 @@ def post_install(spec, explicit=None): tty.debug("SKIP: shebang filtering [external package]") return - install_sbang() - for directory, _, filenames in os.walk(spec.prefix): filter_shebangs_in_directory(directory, filenames) diff --git a/lib/spack/spack/install_test.py b/lib/spack/spack/install_test.py index c38f218a6b829b..5cc542664b255a 100644 --- a/lib/spack/spack/install_test.py +++ b/lib/spack/spack/install_test.py @@ -12,14 +12,13 @@ import shutil import sys from collections import Counter, OrderedDict -from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Type, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Optional, Tuple, Type, Union import spack.config import spack.error import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.llnl.util.tty.log -import spack.package_base import spack.paths import spack.repo import spack.report @@ -34,6 +33,9 @@ from spack.spec import Spec from spack.util.prefix import Prefix +if TYPE_CHECKING: + import spack.package_base + #: Stand-alone test failure info type TestFailureType = Tuple[BaseException, str] @@ -474,9 +476,9 @@ def test_part( wdir = "." if work_dir is None else work_dir tester = pkg.tester - assert test_name and test_name.startswith( - "test_" - ), f"Test name must start with 'test_' but {test_name} was provided" + assert test_name and test_name.startswith("test_"), ( + f"Test name must start with 'test_' but {test_name} was provided" + ) title = "test: {}: {}".format(test_name, purpose or "unspecified purpose") with fs.working_dir(wdir, create=True): @@ -509,9 +511,7 @@ def test_part( if exc_type is spack.util.executable.ProcessError or exc_type is TypeError: iostr = io.StringIO() - write_log_summary( - iostr, "test", tester.test_log_file, last=1 - ) # type: ignore[assignment] + write_log_summary(iostr, "test", tester.test_log_file, last=1) # type: ignore[assignment] m = iostr.getvalue() else: # We're below the package context, so get context from @@ -570,7 +570,7 @@ def copy_test_files(pkg: "spack.package_base.PackageBase", test_spec: spack.spec shutil.copytree(data_source, data_dir) -def test_function_names(pkg: PackageObjectOrClass, add_virtuals: bool = False) -> List[str]: +def test_function_names(pkg: "PackageObjectOrClass", add_virtuals: bool = False) -> List[str]: """Grab the names of all non-empty test functions. Args: @@ -589,7 +589,7 @@ def test_function_names(pkg: PackageObjectOrClass, add_virtuals: bool = False) - def test_functions( - pkg: PackageObjectOrClass, add_virtuals: bool = False + pkg: "PackageObjectOrClass", add_virtuals: bool = False ) -> List[Tuple[str, Callable]]: """Grab all non-empty test functions. @@ -604,12 +604,7 @@ def test_functions( Raises: ValueError: occurs if pkg is not a package class """ - instance = isinstance(pkg, spack.package_base.PackageBase) - if not (instance or issubclass(pkg, spack.package_base.PackageBase)): # type: ignore[arg-type] - raise ValueError(f"Expected a package (class), not {pkg} ({type(pkg)})") - - pkg_cls = pkg.__class__ if instance else pkg - classes = [pkg_cls] + classes = [pkg if isinstance(pkg, type) else pkg.__class__] if add_virtuals: vpkgs = virtuals(pkg) for vname in vpkgs: @@ -863,8 +858,7 @@ def name(self) -> str: def content_hash(self) -> str: """The hash used to uniquely identify the test suite.""" if not self._hash: - json_text = sjson.dump(self.to_dict()) - assert json_text is not None, f"{__name__} unexpected value for 'json_text'" + json_text = sjson.dumps(self.to_dict()) sha = hashlib.sha1(json_text.encode("utf-8")) b32_hash = base64.b32encode(sha.digest()).lower() b32_hash = b32_hash.decode("utf-8") diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py index 23a547007efef7..1c455aa63469bf 100644 --- a/lib/spack/spack/installer.py +++ b/lib/spack/spack/installer.py @@ -68,7 +68,6 @@ from spack.llnl.util.tty.log import log_output, preserve_terminal_settings from spack.url_buildcache import BuildcacheEntryError from spack.util.environment import EnvironmentModifications, dump_environment -from spack.util.executable import which if TYPE_CHECKING: import spack.spec @@ -232,10 +231,6 @@ def _check_last_phase(pkg: "spack.package_base.PackageBase") -> None: if pkg.last_phase and pkg.last_phase not in phases: # type: ignore[attr-defined] raise BadInstallPhase(pkg.name, pkg.last_phase) # type: ignore[attr-defined] - # If we got a last_phase, make sure it's not already last - if pkg.last_phase and pkg.last_phase == phases[-1]: # type: ignore[attr-defined] - pkg.last_phase = None # type: ignore[attr-defined] - def _handle_external_and_upstream(pkg: "spack.package_base.PackageBase", explicit: bool) -> bool: """ @@ -286,10 +281,8 @@ def _do_fake_install(pkg: "spack.package_base.PackageBase") -> None: # Install fake command fs.mkdirp(pkg.prefix.bin) - fs.touch(os.path.join(pkg.prefix.bin, command)) - if sys.platform != "win32": - chmod = which("chmod", required=True) - chmod("+x", os.path.join(pkg.prefix.bin, command)) + executable = lambda path, flags: os.open(path, flags, 0o700) + open(os.path.join(pkg.prefix.bin, command), "wb", opener=executable).close() # Install fake header file fs.mkdirp(pkg.prefix.include) @@ -932,7 +925,8 @@ def __init__( self.request = request # Report for tracking install success/failure - self.record = spack.report.InstallRecord(self.pkg.spec) + record_cls = self.request.install_args.get("record_cls", spack.report.InstallRecord) + self.record = record_cls(self.pkg.spec) # Initialize the status to an active state. The status is used to # ensure priority queue invariants when tasks are "removed" from the @@ -1295,9 +1289,9 @@ def _start_build_process(self): def poll(self): """Check if task has successfully executed, caused an InstallError, or the child process has information ready to receive.""" - assert ( - self.started or self.no_op - ), "Can't call `poll()` before `start()` or identified no-operation task" + assert self.started or self.no_op, ( + "Can't call `poll()` before `start()` or identified no-operation task" + ) return self.no_op or self.success_result or self.error_result or self.process_handle.poll() def succeed(self): @@ -1330,9 +1324,9 @@ def complete(self): Complete the installation of the requested spec and/or dependency represented by the build task. """ - assert ( - self.started or self.no_op - ), "Can't call `complete()` before `start()` or identified no-operation task" + assert self.started or self.no_op, ( + "Can't call `complete()` before `start()` or identified no-operation task" + ) pkg = self.pkg self.status = BuildStatus.INSTALLING @@ -1357,8 +1351,8 @@ def complete(self): self.fail(self.error_result) # hook that allows tests to inspect the Package before installation - # see unit_test_check() docs. - if not pkg.unit_test_check(): + # see _unit_test_check() docs. + if not pkg._unit_test_check(): self.succeed() return ExecuteResult.FAILED @@ -1492,6 +1486,7 @@ def __init__( concurrent_packages: Optional[int] = None, root_policy: InstallPolicy = "auto", dependencies_policy: InstallPolicy = "auto", + create_reports: bool = False, ) -> None: """ Arguments: @@ -1517,6 +1512,7 @@ def __init__( concurrent_packages: Max packages to be built concurrently root_policy: ``"auto"``, ``"cache_only"``, ``"source_only"``. dependencies_policy: ``"auto"``, ``"cache_only"``, ``"source_only"``. + create_reports: whether to generate reports for each install """ if sys.platform == "win32": # No locks on Windows, we should always use 1 process @@ -1529,6 +1525,9 @@ def __init__( if concurrent_packages is None: concurrent_packages = spack.config.get("config:concurrent_packages", default=1) + # The value 0 means no concurrency in the old installer. + if concurrent_packages == 0: + concurrent_packages = 1 self.concurrent_packages = concurrent_packages install_args = { @@ -1558,6 +1557,11 @@ def __init__( # List of build requests self.build_requests = [BuildRequest(pkg, install_args) for pkg in packages] + # When no reporter is configured, use NullInstallRecord to skip log file reads. + if not create_reports: + for br in self.build_requests: + br.install_args["record_cls"] = spack.report.NullInstallRecord + # Priority queue of tasks self.build_pq: List[Tuple[Tuple[int, int], Task]] = [] @@ -1590,12 +1594,17 @@ def __init__( self.max_active_tasks = self.concurrent_packages # Reports on install success/failure - self.reports: Dict[str, spack.report.RequestRecord] = {} - for build_request in self.build_requests: - # Skip reporting for already installed specs - request_record = spack.report.RequestRecord(build_request.pkg.spec) - request_record.skip_installed() - self.reports[build_request.pkg_id] = request_record + if create_reports: + self.reports: Dict[str, spack.report.RequestRecord] = {} + for build_request in self.build_requests: + # Skip reporting for already installed specs + request_record = spack.report.RequestRecord(build_request.pkg.spec) + request_record.skip_installed() + self.reports[build_request.pkg_id] = request_record + else: + self.reports = { + br.pkg_id: spack.report.NullRequestRecord() for br in self.build_requests + } def __repr__(self) -> str: """Returns a formal representation of the package installer.""" @@ -1832,10 +1841,9 @@ def _ensure_locked( Return: (lock_type, lock) tuple where lock will be None if it could not be obtained """ - assert lock_type in [ - "read", - "write", - ], f'"{lock_type}" is not a supported package management lock type' + assert lock_type in ["read", "write"], ( + f'"{lock_type}" is not a supported package management lock type' + ) pkg_id = package_id(pkg.spec) ltype, lock = self.locks.get(pkg_id, (lock_type, None)) @@ -2371,9 +2379,7 @@ def complete_task(self, task: Task, install_status: InstallStatus) -> Optional[T except KeyboardInterrupt as exc: # The build has been terminated with a Ctrl-C so terminate # regardless of the number of remaining specs. - tty.error( - f"Failed to install {pkg.name} due to " f"{exc.__class__.__name__}: {str(exc)}" - ) + tty.error(f"Failed to install {pkg.name} due to {exc.__class__.__name__}: {str(exc)}") raise except BuildcacheEntryError as exc: @@ -2414,7 +2420,7 @@ def complete_task(self, task: Task, install_status: InstallStatus) -> Optional[T # lower levels -- skip printing if already printed. # TODO: sort out this and SpackError.print_context() tty.error( - f"Failed to install {pkg.name} due to " f"{exc.__class__.__name__}: {str(exc)}" + f"Failed to install {pkg.name} due to {exc.__class__.__name__}: {str(exc)}" ) # Terminate if requested to do so on the first failure. if self.fail_fast: @@ -2455,6 +2461,7 @@ def _install(self) -> None: """ + spack.store.STORE.install_sbang() self._init_queue() failed_build_requests = [] install_status = InstallStatus(len(self.build_pq)) @@ -2515,7 +2522,7 @@ def _install(self) -> None: # be dependencies of this task. term_status.clear() tty.error( - f"Detected uninstalled dependencies for {task.pkg_id}: " f"{task.uninstalled_deps}" + f"Detected uninstalled dependencies for {task.pkg_id}: {task.uninstalled_deps}" ) left = [dep_id for dep_id in task.uninstalled_deps if dep_id not in self.installed] if not left: @@ -2736,7 +2743,7 @@ def _real_install(self) -> None: # DEBUGGING TIP - to debug this section, insert an IPython # embed here, and run the sections below without log capture log_contextmanager = log_output( - log_file, self.echo, True, filter_fn=self.filter_fn + log_file, self.echo, debug=True, filter_fn=self.filter_fn ) with log_contextmanager as logger: diff --git a/lib/spack/spack/installer_dispatch.py b/lib/spack/spack/installer_dispatch.py new file mode 100644 index 00000000000000..0d48a38aa07ede --- /dev/null +++ b/lib/spack/spack/installer_dispatch.py @@ -0,0 +1,95 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import sys +from typing import TYPE_CHECKING, List, Optional, Set, Union + +from spack.vendor.typing_extensions import Literal + +import spack.config +import spack.sandbox +import spack.traverse + +if TYPE_CHECKING: + import spack.installer + import spack.new_installer + import spack.package_base + + +def create_installer( + packages: List["spack.package_base.PackageBase"], + *, + dirty: bool = False, + explicit: Union[Set[str], bool] = False, + overwrite: Optional[Union[List[str], Set[str]]] = None, + fail_fast: bool = False, + fake: bool = False, + include_build_deps: bool = False, + install_deps: bool = True, + install_package: bool = True, + install_source: bool = False, + keep_prefix: bool = False, + keep_stage: bool = False, + restage: bool = True, + skip_patch: bool = False, + stop_at: Optional[str] = None, + stop_before: Optional[str] = None, + tests: Union[bool, List[str], Set[str]] = False, + unsigned: Optional[bool] = None, + verbose: bool = False, + concurrent_packages: Optional[int] = None, + root_policy: Literal["auto", "cache_only", "source_only"] = "auto", + dependencies_policy: Literal["auto", "cache_only", "source_only"] = "auto", + create_reports: bool = False, +) -> Union["spack.installer.PackageInstaller", "spack.new_installer.PackageInstaller"]: + """Create an installer based on the current configuration and feature support.""" + use_old_installer = ( + sys.platform == "win32" or spack.config.get("config:installer", "new") == "old" + ) + + # Use the old installer if splicing is used. + if not use_old_installer: + specs = [pkg.spec for pkg in packages] + for s in spack.traverse.traverse_nodes(specs): + if s.build_spec is not s: + use_old_installer = True + break + if spack.config.get("config:sandbox:enable", False): + if use_old_installer: + raise spack.sandbox.SandboxError( + "config:sandbox:enable is only supported with config:installer:new" + ) + # Probe sandbox support now so builds don't fail later inside a subprocess. + spack.sandbox.get_sandbox() + + if use_old_installer: + from spack.installer import PackageInstaller # type: ignore + else: + from spack.new_installer import PackageInstaller # type: ignore + + return PackageInstaller( + packages, + dirty=dirty, + explicit=explicit, + overwrite=overwrite, + fail_fast=fail_fast, + fake=fake, + include_build_deps=include_build_deps, + install_deps=install_deps, + install_package=install_package, + install_source=install_source, + keep_prefix=keep_prefix, + keep_stage=keep_stage, + restage=restage, + skip_patch=skip_patch, + stop_at=stop_at, + stop_before=stop_before, + tests=tests, + unsigned=unsigned, + verbose=verbose, + concurrent_packages=concurrent_packages, + root_policy=root_policy, + dependencies_policy=dependencies_policy, + create_reports=create_reports, + ) diff --git a/lib/spack/spack/llnl/path.py b/lib/spack/spack/llnl/path.py index 30b9f2f24e2533..11dff3c92ba0fe 100644 --- a/lib/spack/spack/llnl/path.py +++ b/lib/spack/spack/llnl/path.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Path primitives that just require Python standard library.""" + import functools import sys from typing import List, Optional diff --git a/lib/spack/spack/llnl/string.py b/lib/spack/spack/llnl/string.py index 2f15051bf9634a..472b7ff1ac0150 100644 --- a/lib/spack/spack/llnl/string.py +++ b/lib/spack/spack/llnl/string.py @@ -4,10 +4,11 @@ """String manipulation functions that do not have other dependencies than Python standard library """ -from typing import List, Optional +from typing import List, Optional, Sequence -def comma_list(sequence: List[str], article: str = "") -> str: + +def comma_list(sequence: Sequence[str], article: str = "") -> str: if type(sequence) is not list: sequence = list(sequence) @@ -26,7 +27,7 @@ def comma_list(sequence: List[str], article: str = "") -> str: return out -def comma_or(sequence: List[str]) -> str: +def comma_or(sequence: Sequence[str]) -> str: """Return a string with all the elements of the input joined by comma, but the last one (which is joined by ``"or"``). """ diff --git a/lib/spack/spack/llnl/url.py b/lib/spack/spack/llnl/url.py index 6abc0552df35df..b5d2c3c10e5d02 100644 --- a/lib/spack/spack/llnl/url.py +++ b/lib/spack/spack/llnl/url.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """URL primitives that just require Python standard library.""" + import itertools import os import re @@ -90,11 +91,9 @@ def find_list_urls(url: str) -> Set[str]: ( r"luarocks[^/]+/(?:modules|manifests)/(?P[^/]+)/" + r"(?P.+?)-[0-9.-]*\.src\.rock", - lambda m: "https://luarocks.org/modules/" - + m.group("org") - + "/" - + m.group("name") - + "/", + lambda m: ( + "https://luarocks.org/modules/" + m.group("org") + "/" + m.group("name") + "/" + ), ), ] @@ -223,7 +222,7 @@ def split_url_extension(url: str) -> Tuple[str, ...]: 1. ``('https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7', '.tgz', '?raw=true')`` 2. ``('http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin', '.tar.gz', None)`` 3. ``('https://gitlab.kitware.com/vtk/vtk/repository/archive', '.tar.bz2', '?ref=v7.0.0')`` - """ + """ # noqa: E501 # Strip off sourceforge download suffix. # e.g. https://sourceforge.net/projects/glew/files/glew/2.0.0/glew-2.0.0.tgz/download prefix, suffix = split_url_on_sourceforge_suffix(url) diff --git a/lib/spack/spack/llnl/util/argparsewriter.py b/lib/spack/spack/llnl/util/argparsewriter.py index 8771b32e40f490..ef1a27c24e48ed 100644 --- a/lib/spack/spack/llnl/util/argparsewriter.py +++ b/lib/spack/spack/llnl/util/argparsewriter.py @@ -263,9 +263,7 @@ def begin_command(self, prog: str) -> str: {1} {2} -""".format( - prog.replace(" ", "-"), prog, self.rst_levels[self.level] * len(prog) - ) +""".format(prog.replace(" ", "-"), prog, self.rst_levels[self.level] * len(prog)) def description(self, description: str) -> str: """Description of a command. @@ -292,9 +290,7 @@ def usage(self, usage: str) -> str: {0} -""".format( - usage - ) +""".format(usage) def begin_positionals(self) -> str: """Text to print before positional arguments. @@ -318,9 +314,7 @@ def positional(self, name: str, help: str) -> str: ``{0}`` {1} -""".format( - name, help - ) +""".format(name, help) def end_positionals(self) -> str: """Text to print after positional arguments. @@ -352,9 +346,7 @@ def optional(self, opts: str, help: str) -> str: ``{0}`` {1} -""".format( - opts, help - ) +""".format(opts, help) def end_optionals(self) -> str: """Text to print after optional arguments. diff --git a/lib/spack/spack/llnl/util/filesystem.py b/lib/spack/spack/llnl/util/filesystem.py index 1ddfa6a093c30a..8ce13132d1078f 100644 --- a/lib/spack/spack/llnl/util/filesystem.py +++ b/lib/spack/spack/llnl/util/filesystem.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections import collections.abc +import ctypes import errno import fnmatch import glob @@ -41,12 +42,11 @@ from spack.llnl.util import lang, tty from spack.llnl.util.lang import dedupe, fnmatch_translate_multiple, memoized -if sys.platform != "win32": +if sys.platform == "win32": + from ctypes import wintypes +else: import grp import pwd -else: - import win32security - from win32file import CreateHardLink __all__ = [ @@ -201,18 +201,17 @@ def polite_filename(filename: str) -> str: if sys.platform == "win32": - def _getuid_win32() -> Union[str, int]: + def _getuid_win32() -> str: """Returns os getuid on non Windows On Windows returns 0 for admin users, login string otherwise This is in line with behavior from get_owner_uid which always returns the login string on Windows """ - import ctypes # If not admin, use the string name of the login as a unique ID if ctypes.windll.shell32.IsUserAnAdmin() == 0: return os.getlogin() - return 0 + return "ADMINISTRATORS" getuid = _getuid_win32 else: @@ -534,6 +533,129 @@ def exploding_archive_handler(tarball_container, stage): shutil.move(tarball_container, stage.source_path) +@system_path_filter +def get_windows_file_security(path: str) -> str: + if sys.platform == "win32": + # Validate path exists before calling API to get a clear Python error + if not os.path.exists(path): + raise FileNotFoundError(f"The system cannot find the path specified: '{path}'") + + advapi = ctypes.WinDLL("advapi32", use_last_error=True) + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + + # SE_FILE_OBJECT applies to both files and directories + SE_FILE_OBJECT = 1 + OWNER_SECURITY_INFO = 1 + ERROR_SUCCESS = 0 + + # Describe LocalFree API + LocalFree = kernel32.LocalFree + LocalFree.argtypes = [wintypes.HLOCAL] + LocalFree.restype = wintypes.HLOCAL + + # Describe GetNamedSecurityInfoW API + GetNamedSecurityInfo = advapi.GetNamedSecurityInfoW + GetNamedSecurityInfo.argtypes = [ + wintypes.LPCWSTR, # pObjectName (The path) + ctypes.c_int, # ObjectType + wintypes.DWORD, # SecurityInfo + ctypes.POINTER(wintypes.LPVOID), # ppsidOwner + ctypes.POINTER(wintypes.LPVOID), # ppsidGroup + ctypes.POINTER(wintypes.LPVOID), # ppDacl + ctypes.POINTER(wintypes.LPVOID), # ppSacl + ctypes.POINTER(wintypes.LPVOID), # ppSecurityDescriptor + ] + GetNamedSecurityInfo.restype = wintypes.DWORD + + # Describe LookupAccountSID API + LookupAccountSid = advapi.LookupAccountSidW + LookupAccountSid.argtypes = [ + wintypes.LPCWSTR, # lpSystemName + wintypes.LPVOID, # Sid + wintypes.LPWSTR, # Name + wintypes.LPDWORD, # cchName + wintypes.LPWSTR, # ReferencedDomainName + wintypes.LPDWORD, # cchReferencedDomainName + ctypes.POINTER(ctypes.c_int), # peUse + ] + LookupAccountSid.restype = ctypes.c_bool + + p_sid_owner = wintypes.LPVOID() + psd = wintypes.LPVOID() + + # Call GetNamedSecurityInfo directly with the path string + res = GetNamedSecurityInfo( + path, + SE_FILE_OBJECT, + OWNER_SECURITY_INFO, + ctypes.byref(p_sid_owner), + None, + None, + None, + ctypes.byref(psd), + ) + + if res != ERROR_SUCCESS: + raise ctypes.WinError(res, f"Failed to get security info for {path}") + + try: + # establish vars for Lookup account sid return params + dwacct_name = wintypes.DWORD(0) + dw_domain_name = wintypes.DWORD(0) + e_use = ctypes.c_int() + + # first call to lookup account SID to determine buffer sizes + success = LookupAccountSid( + None, + p_sid_owner, + None, + ctypes.byref(dwacct_name), + None, + ctypes.byref(dw_domain_name), + ctypes.byref(e_use), + ) + + # 122 is ERROR_INSUFFICIENT_BUFFER, which we expect + if not success: + err = ctypes.get_last_error() + # 122 is ERROR_INSUFFICIENT_BUFFER, which we want/expect! + if err != 122: + raise ctypes.WinError( + err, f"Unexpected error when obtaining buffer for : {path}" + ) + + # create buffers + acct_name_buf = dwacct_name.value * wintypes.WCHAR + acct_name = acct_name_buf() + domain_name_buf = dw_domain_name.value * wintypes.WCHAR + domain_name = domain_name_buf() + + # second call to fetch the actual names + success = LookupAccountSid( + None, + p_sid_owner, + acct_name, + ctypes.byref(dwacct_name), + domain_name, + ctypes.byref(dw_domain_name), + ctypes.byref(e_use), + ) + + if not success: + raise ctypes.WinError( + ctypes.get_last_error(), f"Could not determine owner for : {path}" + ) + + finally: + # Free the security descriptor + if psd: + LocalFree(psd) + + return acct_name.value + else: + raise RuntimeError("cannot determine Windows file security on non-Windows") + + @system_path_filter(arg_slice=slice(1)) def get_owner_uid(path, err_msg=None) -> Union[str, int]: """Returns owner UID of path destination @@ -560,10 +682,7 @@ def get_owner_uid(path, err_msg=None) -> Union[str, int]: if sys.platform != "win32": owner_uid = p_stat.st_uid else: - sid = win32security.GetFileSecurity( - path, win32security.OWNER_SECURITY_INFORMATION - ).GetSecurityDescriptorOwner() - owner_uid = win32security.LookupAccountSid(None, sid)[0] + owner_uid = get_windows_file_security(path) return owner_uid @@ -1213,19 +1332,17 @@ def windows_sfn(path: os.PathLike): path: Path to be transformed into SFN (8.3 filename) format """ # This should not be run-able on linux/macos - if sys.platform != "win32": - return path - path = str(path) - import ctypes - - k32 = ctypes.WinDLL("kernel32", use_last_error=True) - # Method with null values returns size of short path name - sz = k32.GetShortPathNameW(path, None, 0) - # stub Windows types TCHAR[LENGTH] - TCHAR_arr = ctypes.c_wchar * sz - ret_str = TCHAR_arr() - k32.GetShortPathNameW(path, ctypes.byref(ret_str), sz) - return ret_str.value + if sys.platform == "win32": + path = str(path) + k32 = ctypes.WinDLL("kernel32", use_last_error=True) + # Method with null values returns size of short path name + sz = k32.GetShortPathNameW(path, None, 0) + # stub Windows types TCHAR[LENGTH] + TCHAR_arr = ctypes.c_wchar * sz + ret_str = TCHAR_arr() + k32.GetShortPathNameW(path, ctypes.byref(ret_str), sz) + return ret_str.value + return path @contextmanager @@ -1875,7 +1992,6 @@ def _find_max_depth( with dir_iter: for dir_entry in dir_iter: - # Match filename only patterns if filename_only_patterns: m = regex.match(os.path.normcase(dir_entry.name)) @@ -2902,7 +3018,7 @@ def _windows_symlink( src: str, dst: str, target_is_directory: bool = False, *, dir_fd: Union[int, None] = None ): """On Windows with System Administrator privileges this will be a normal symbolic link via - os.symlink. On Windows without privledges the link will be a junction for a directory and a + os.symlink. On Windows without privileges the link will be a junction for a directory and a hardlink for a file. On Windows the various link types are: Symbolic Link: A link to a file or directory on the same or different volume (drive letter) or @@ -3004,23 +3120,23 @@ def _windows_is_junction(path: str) -> bool: Returns: bool - whether the path is a junction or not. """ - if sys.platform != "win32" or os.path.islink(path) or os.path.isfile(path): - return False - - import ctypes.wintypes + if sys.platform == "win32": + if os.path.islink(path) or os.path.isfile(path): + return False - get_file_attributes = ctypes.windll.kernel32.GetFileAttributesW # type: ignore[attr-defined] - get_file_attributes.argtypes = (ctypes.wintypes.LPWSTR,) - get_file_attributes.restype = ctypes.wintypes.DWORD + get_file_attributes = ctypes.windll.kernel32.GetFileAttributesW # type: ignore[attr-defined] + get_file_attributes.argtypes = (wintypes.LPWSTR,) + get_file_attributes.restype = wintypes.DWORD - invalid_file_attributes = 0xFFFFFFFF - reparse_point = 0x400 - file_attr = get_file_attributes(str(path)) + invalid_file_attributes = 0xFFFFFFFF + reparse_point = 0x400 + file_attr = get_file_attributes(str(path)) - if file_attr == invalid_file_attributes: - return False + if file_attr == invalid_file_attributes: + return False - return file_attr & reparse_point > 0 + return file_attr & reparse_point > 0 + return False @lang.memoized @@ -3119,7 +3235,16 @@ def _windows_create_hard_link(path: str, link: str): raise SymlinkError(f"File path ({link}) is not a file. Cannot create hard link.") else: tty.debug(f"Creating hard link {link} pointing to {path}") - CreateHardLink(link, path) + k32 = ctypes.WinDLL("kernel32", use_last_error=True) + CreateHardLink = k32.CreateHardLinkW + CreateHardLink.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_void_p] + CreateHardLink.restype = ctypes.c_bool + success = CreateHardLink(link, path, None) + if not success: + error_code = ctypes.GetLastError() + raise ctypes.WinError( + error_code, f"Failed to create hardlink for path {path} and link {link}" + ) def _windows_readlink(path: str, *, dir_fd=None): diff --git a/lib/spack/spack/llnl/util/lang.py b/lib/spack/spack/llnl/util/lang.py index f12c2d7e47f8ce..b0fada151ae35f 100644 --- a/lib/spack/spack/llnl/util/lang.py +++ b/lib/spack/spack/llnl/util/lang.py @@ -211,7 +211,7 @@ def setter(name, value): def tuplify(seq): """Helper for lazy_lexicographic_ordering().""" - return tuple((tuplify(x) if callable(x) else x) for x in seq()) + return tuple([(tuplify(x) if callable(x) else x) for x in seq()]) def lazy_eq(lseq, rseq): @@ -456,7 +456,7 @@ def __delitem__(self, key: K) -> None: del self.dict[key] def _cmp_iter(self): - for _, v in sorted(self.items()): + for _, v in sorted(self.dict.items()): yield v @@ -487,7 +487,7 @@ def match(string): return True else: raise ValueError( - "args to match_predicate must be regex, " "list of regexes, or callable." + "args to match_predicate must be regex, list of regexes, or callable." ) return False @@ -670,6 +670,19 @@ def pretty_seconds(seconds): return pretty_seconds_formatter(seconds)(seconds) +def pretty_duration(seconds: float) -> str: + """Format a duration in seconds as a compact human-readable string (e.g. "1h02m", "3m05s", + "45s").""" + s = int(seconds) + if s < 60: + return f"{s}s" + m, s = divmod(s, 60) + if m < 60: + return f"{m}m{s:02d}s" + h, m = divmod(m, 60) + return f"{h}h{m:02d}m" + + class ObjectWrapper: """Base class that wraps an object. Derived classes can add new behavior while staying undercover. @@ -723,16 +736,23 @@ def __init__(self, factory: Callable[[], object]): @property def instance(self): if self._instance is None: - instance = self.factory() - - if isinstance(instance, types.GeneratorType): - # if it's a generator, assign every value - for value in instance: - self._instance = value - else: - # if not, just assign the result like a normal singleton - self._instance = instance - + try: + instance = self.factory() + + if isinstance(instance, types.GeneratorType): + # if it's a generator, assign every value + for value in instance: + self._instance = value + else: + # if not, just assign the result like a normal singleton + self._instance = instance + except AttributeError as e: + # getattr will "absorb" an AttributeError that occurs + # during the execution of the factory method: we'd like + # to show that so wrap it in something that isn't absorbed + raise SingletonInstantiationError( + "AttrbuteError during creation of Singleton instance" + ) from e return self._instance def __getattr__(self, name): @@ -763,6 +783,10 @@ def __repr__(self): return repr(self.instance) +class SingletonInstantiationError(Exception): + """Error that indicates a singleton that cannot instantiate.""" + + def get_entry_points(*, group: str): """Wrapper for ``importlib.metadata.entry_points`` diff --git a/lib/spack/spack/llnl/util/link_tree.py b/lib/spack/spack/llnl/util/link_tree.py index bb302ef5d1add3..4eb738e0eea7c6 100644 --- a/lib/spack/spack/llnl/util/link_tree.py +++ b/lib/spack/spack/llnl/util/link_tree.py @@ -7,7 +7,8 @@ import filecmp import os import shutil -from typing import Callable, Dict, List, Optional, Tuple +from pathlib import Path +from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty @@ -51,22 +52,41 @@ def _samefile(a: str, b: str): return False -class SourceMergeVisitor(fs.BaseDirectoryVisitor): - """ - Visitor that produces actions: - - An ordered list of directories to create in dst - - A list of files to link in dst - - A list of merge conflicts in dst/ - """ +#: (index, src_root, rel_path, is_symlink) +FileEntry = Tuple[int, str, str, bool] + +#: (index, src_root, rel_path) +DirEntry = Tuple[int, str, str] + +PrefixAndProjection = Union[Union[str, Path], Tuple[Union[str, Path], Union[str, Path]]] + + +class MultiPrefixMerger: + """Class that takes multiple pairs of prefixes and projections, and produces a list of + directories to create, files to link, and conflicts when merging them together.""" def __init__( - self, ignore: Optional[Callable[[str], bool]] = None, normalize_paths: bool = False + self, + sources: Sequence[PrefixAndProjection], + ignore: Optional[Callable[[str], bool]] = None, + normalize_paths: bool = False, + dir_symlink_optimization: bool = False, ): + """ + Args: + sources: list of source directories, or tuples of (source directory, projection) pairs + ignore: optional callable(rel_path) -> bool to skip entries + normalize_paths: whether to normalize paths for case-insensitive filesystems + dir_symlink_optimization: whether to enable directory-level symlink optimization + """ self.ignore = ignore if ignore is not None else lambda f: False # On case-insensitive filesystems, normalize paths to detect duplications self.normalize_paths = normalize_paths + #: Whether to symlink directories unique to one source + self._dir_symlink_optimization = dir_symlink_optimization + # When mapping to /, we need to prepend the # bit to the relative path in the destination dir. self.projection: str = "" @@ -97,6 +117,21 @@ def __init__( # normalized path to: original path, root directory + relative path self._files_normalized: Dict[str, Tuple[str, str, str]] = {} + # Group sources by projection + projection_groups: Dict[str, List[str]] = {} + for src in sources: + if isinstance(src, tuple): + src_root, projection = src + else: + src_root, projection = src, "" + projection_groups.setdefault(str(projection), []).append(str(src_root)) + + # Process each projection group + for projection, roots in projection_groups.items(): + self.set_projection(projection) + active = [(i, root, "") for i, root in enumerate(roots)] + self._simultaneous_recurse(active, 0) + def _in_directories(self, proj_rel_path: str) -> bool: """ Check if a path is already in the directory list @@ -150,14 +185,6 @@ def _file(self, proj_rel_path: str) -> Tuple[str, str, str]: else: return (proj_rel_path, *self.files[proj_rel_path]) - def _del_file(self, proj_rel_path: str): - """ - Remove a file from the list of files - """ - del self.files[proj_rel_path] - if self.normalize_paths: - del self._files_normalized[proj_rel_path.lower()] - def _add_file(self, proj_rel_path: str, root: str, rel_path: str): """ Add a file to the list of files @@ -167,113 +194,6 @@ def _add_file(self, proj_rel_path: str, root: str, rel_path: str): if self.normalize_paths: self._files_normalized[proj_rel_path.lower()] = (proj_rel_path, root, rel_path) - def before_visit_dir(self, root: str, rel_path: str, depth: int) -> bool: - """ - Register a directory if dst / rel_path is not blocked by a file or ignored. - """ - proj_rel_path = os.path.join(self.projection, rel_path) - - if self.ignore(rel_path): - # Don't recurse when dir is ignored. - return False - elif self._in_files(proj_rel_path): - # A file-dir conflict is fatal except if they're the same file (symlinked dir). - src_a = os.path.join(*self._file(proj_rel_path)) - src_b = os.path.join(root, rel_path) - - if not _samefile(src_a, src_b): - self.fatal_conflicts.append( - MergeConflict(dst=proj_rel_path, src_a=src_a, src_b=src_b) - ) - return False - - # Remove the link in favor of the dir. - existing_proj_rel_path, _, _ = self._file(proj_rel_path) - self._del_file(existing_proj_rel_path) - self._add_directory(proj_rel_path, root, rel_path) - return True - elif self._in_directories(proj_rel_path): - # No new directory, carry on. - return True - else: - # Register new directory. - self._add_directory(proj_rel_path, root, rel_path) - return True - - def before_visit_symlinked_dir(self, root: str, rel_path: str, depth: int) -> bool: - """ - Replace symlinked dirs with actual directories when possible in low depths, - otherwise handle it as a file (i.e. we link to the symlink). - - Transforming symlinks into dirs makes it more likely we can merge directories, - e.g. when /lib -> /subdir/lib. - - We only do this when the symlink is pointing into a subdirectory from the - symlink's directory, to avoid potential infinite recursion; and only at a - constant level of nesting, to avoid potential exponential blowups in file - duplication. - """ - if self.ignore(rel_path): - return False - - # Only follow symlinked dirs in /**/**/* - if depth > 1: - handle_as_dir = False - else: - # Only follow symlinked dirs when pointing deeper - src = os.path.join(root, rel_path) - real_parent = os.path.realpath(os.path.dirname(src)) - real_child = os.path.realpath(src) - handle_as_dir = real_child.startswith(real_parent) - - if handle_as_dir: - return self.before_visit_dir(root, rel_path, depth) - - self.visit_file(root, rel_path, depth, symlink=True) - return False - - def visit_file(self, root: str, rel_path: str, depth: int, *, symlink: bool = False) -> None: - proj_rel_path = os.path.join(self.projection, rel_path) - - if self.ignore(rel_path): - pass - elif self._in_directories(proj_rel_path): - # Can't create a file where a dir is, unless they are the same file (symlinked dir), - # in which case we simply drop the symlink in favor of the actual dir. - src_a = os.path.join(*self._directory(proj_rel_path)) - src_b = os.path.join(root, rel_path) - if not symlink or not _samefile(src_a, src_b): - self.fatal_conflicts.append( - MergeConflict(dst=proj_rel_path, src_a=src_a, src_b=src_b) - ) - elif self._in_files(proj_rel_path): - # When two files project to the same path, they conflict iff they are distinct. - # If they are the same (i.e. one links to the other), register regular files rather - # than symlinks. The reason is that in copy-type views, we need a copy of the actual - # file, not the symlink. - src_a = os.path.join(*self._file(proj_rel_path)) - src_b = os.path.join(root, rel_path) - if not _samefile(src_a, src_b): - # Distinct files produce a conflict. - self.file_conflicts.append( - MergeConflict(dst=proj_rel_path, src_a=src_a, src_b=src_b) - ) - return - - if not symlink: - # Remove the link in favor of the actual file. The del is necessary to maintain the - # order of the files dict, which is grouped by root. - existing_proj_rel_path, _, _ = self._file(proj_rel_path) - self._del_file(existing_proj_rel_path) - self._add_file(proj_rel_path, root, rel_path) - else: - # Otherwise register this file to be linked. - self._add_file(proj_rel_path, root, rel_path) - - def visit_symlinked_file(self, root: str, rel_path: str, depth: int) -> None: - # Treat symlinked files as ordinary files (without "dereferencing") - self.visit_file(root, rel_path, depth, symlink=True) - def set_projection(self, projection: str) -> None: self.projection = os.path.normpath(projection) @@ -300,9 +220,156 @@ def set_projection(self, projection: str) -> None: ) ) + def _simultaneous_recurse(self, active: List[Tuple[int, str, str]], depth: int) -> None: + """Recursively scan active sources simultaneously. + + Args: + depth: current depth from source root (for symlinked dir handling) + active: list of (index, src_root, rel_path) tuples that have this directory + """ + # Mapping of normalized entry names to their corresponding directory and file entries + entry_map: Dict[str, Tuple[List[DirEntry], List[FileEntry]]] = {} + + for idx, src_root, rel_path in active: + scan_path = os.path.join(src_root, rel_path) if rel_path else src_root + try: + scanner = os.scandir(scan_path) + except OSError: + continue # skip if we cannot list directory entries. + + with scanner: + for dir_entry in scanner: + name = dir_entry.name + child_rel = os.path.join(rel_path, name) if rel_path else name + + if self.ignore(child_rel): + continue + + is_link = dir_entry.is_symlink() + try: + is_dir = dir_entry.is_dir(follow_symlinks=True) + except OSError: + is_dir = False # broken symlink is not a dir. + + norm_name = name.lower() if self.normalize_paths else name + dirs, files = entry_map.setdefault(norm_name, ([], [])) + + if is_dir and not is_link: + dirs.append((idx, src_root, child_rel)) + elif is_dir and is_link: + if self._should_follow_symlinked_dir(src_root, child_rel, depth): + dirs.append((idx, src_root, child_rel)) + else: + files.append((idx, src_root, child_rel, True)) + else: + files.append((idx, src_root, child_rel, is_link)) + + # Process collected entries in sorted order + for norm_name in sorted(entry_map): + dirs, files = entry_map[norm_name] + + # When dirs and files project to the same path, we have a potential fatal conflict. + if dirs and files: + rel_path = dirs[0][2] + dir_proj = os.path.join(self.projection, rel_path) if self.projection else rel_path + conflicts = self._dir_file_conflicts(dir_proj, dirs, files) + + if not conflicts: + # all files were symlinks to a dir at the same projected location, ignore them. + files.clear() + else: + # actual dir-file conflicts we cannot resolve. + self.fatal_conflicts.extend(conflicts) + continue + + # Note: no elif. We now have either files or dirs. + if files: + self._handle_files(files) + elif dirs and self._handle_dirs(dirs, depth): + self._simultaneous_recurse(dirs, depth + 1) + + def _should_follow_symlinked_dir(self, src_root: str, rel_path: str, depth: int) -> bool: + """Determine if a symlinked directory should be followed (treated as real dir) + or treated as a file.""" + if depth > 1: + return False + src = os.path.join(src_root, rel_path) + real_parent = os.path.realpath(os.path.dirname(src)) + real_child = os.path.realpath(src) + return real_child.startswith(real_parent) + + def _handle_files(self, files: List[FileEntry]) -> None: + """Handle file entries that all map to the same projected path.""" + # In case of resolvable conflicts (conflicting files are links to the same file) + # the best candidate for the source is the non-symlink file. + + _, root, rel_path, is_symlink = files[0] + dst = os.path.join(self.projection, rel_path) if self.projection else rel_path + for _, other_root, other_rel_path, other_is_symlink in files[1:]: + first_path = os.path.join(root, rel_path) + other_path = os.path.join(other_root, other_rel_path) + if not _samefile(first_path, other_path): + # two distinct files project to the same path; this is a conflict. + self.file_conflicts.append( + MergeConflict(dst=dst, src_a=first_path, src_b=other_path) + ) + elif not other_is_symlink and is_symlink: + # if they are the same, prefer the non-symlink as the source. + root, rel_path, is_symlink = other_root, other_rel_path, other_is_symlink + dst = os.path.join(self.projection, rel_path) if self.projection else rel_path + + self._add_file(dst, root, rel_path) + + def _handle_dirs(self, dirs: List[DirEntry], depth: int) -> bool: + """Handle directory entries that all map to the same projected path. + + Returns True if the caller should recurse deeper into this directory. + """ + _, src_root, rel_path = dirs[0] + proj_child = os.path.join(self.projection, rel_path) if self.projection else rel_path + if self._dir_symlink_optimization and depth > 0 and len(dirs) == 1: + # Unique subtree optimization: if this directory is unique to one source, and we're + # using symlinks, and we're not at the root level, we simply symlink the directory + # rather than creating it in the view and recursing into it. + self._add_file(proj_child, src_root, rel_path) + return False + else: + # Subtree optimization not possible, register make dirs operations and recurse. + self._add_directory(proj_child, src_root, rel_path) + return True + + def _dir_file_conflicts( + self, proj_child_rel: str, dirs: List[DirEntry], files: List[FileEntry] + ) -> Optional[List[MergeConflict]]: + """Handle dir-file conflicts at the same projected path.""" + # We drop all symlinks that resolve to any of the directories that project to the same path + # For example the symlink `/include -> /include` is a resolvable + # conflict as we just keep `/include` in the view. Notice that this is a very rare + # occurrence. + remaining_files = [ + os.path.join(file_root, file_rel_path) + for _, file_root, file_rel_path, is_sym in files + if not is_sym + or not any( + _samefile( + os.path.join(file_root, file_rel_path), os.path.join(dir_root, dir_rel_path) + ) + for _, dir_root, dir_rel_path in dirs + ) + ] + if not remaining_files: + return None + # Use the first dir is the representative dir to register conflicts. + _, src_root, rel_path = dirs[0] + dir_src = os.path.join(src_root, rel_path) + return [ + MergeConflict(dst=proj_child_rel, src_a=dir_src, src_b=file_path) + for file_path in remaining_files + ] + class DestinationMergeVisitor(fs.BaseDirectoryVisitor): - """DestinationMergeVisitor takes a SourceMergeVisitor and: + """DestinationMergeVisitor takes a MultiPrefixMerger and: a. registers additional conflicts when merging to the destination prefix b. removes redundant mkdir operations when directories already exist in the destination prefix. @@ -311,7 +378,7 @@ class DestinationMergeVisitor(fs.BaseDirectoryVisitor): directories in the sources directories. """ - def __init__(self, source_merge_visitor: SourceMergeVisitor): + def __init__(self, source_merge_visitor: MultiPrefixMerger): self.src = source_merge_visitor def before_visit_dir(self, root: str, rel_path: str, depth: int) -> bool: diff --git a/lib/spack/spack/llnl/util/lock.py b/lib/spack/spack/llnl/util/lock.py index 63abafa6ece8aa..dfecfbbf5f6613 100644 --- a/lib/spack/spack/llnl/util/lock.py +++ b/lib/spack/spack/llnl/util/lock.py @@ -9,7 +9,7 @@ import time from datetime import datetime from types import TracebackType -from typing import IO, Any, Callable, ContextManager, Dict, Generator, Optional, Tuple, Type, Union +from typing import IO, Callable, Dict, Generator, Optional, Tuple, Type from spack.llnl.util import lang, tty @@ -34,7 +34,12 @@ ] -ReleaseFnType = Optional[Callable[[], bool]] +ExitFnType = Callable[ + [Optional[Type[BaseException]], Optional[BaseException], Optional[TracebackType]], + Optional[bool], +] +ReleaseFnType = Optional[Callable[[], Optional[bool]]] +DevIno = Tuple[int, int] # (st_dev, st_ino) from os.stat_result def true_fn() -> bool: @@ -43,119 +48,92 @@ def true_fn() -> bool: class OpenFile: - """Record for keeping track of open lockfiles (with reference counting). + """Record for keeping track of open lockfiles (with reference counting).""" - There's really only one ``OpenFile`` per inode, per process, but we record the - filehandle here as it's the thing we end up using in python code. You can get - the file descriptor from the file handle if needed -- or we could make this track - file descriptors as well in the future. - """ + __slots__ = ("fh", "key", "refs") - def __init__(self, fh: IO) -> None: + def __init__(self, fh: IO[bytes], key: DevIno): self.fh = fh + self.key = key # (dev, ino) self.refs = 0 class OpenFileTracker: - """Track open lockfiles, to minimize number of open file descriptors. - - The ``fcntl`` locks that Spack uses are associated with an inode and a process. - This is convenient, because if a process exits, it releases its locks. - Unfortunately, this also means that if you close a file, *all* locks associated - with that file's inode are released, regardless of whether the process has any - other open file descriptors on it. - - Because of this, we need to track open lock files so that we only close them when - a process no longer needs them. We do this by tracking each lockfile by its - inode and process id. This has several nice properties: - - 1. Tracking by pid ensures that, if we fork, we don't inadvertently track the parent - process's lockfiles. ``fcntl`` locks are not inherited across forks, so we'll - just track new lockfiles in the child. - 2. Tracking by inode ensures that references are counted per inode, and that we don't - inadvertently close a file whose inode still has open locks. - 3. Tracking by both pid and inode ensures that we only open lockfiles the minimum - number of times necessary for the locks we have. - - Note: as mentioned elsewhere, these locks aren't thread safe -- they're designed to - work in Python and assume the GIL. - """ + """Track open lockfiles by inode, to minimize the number of open file descriptors. - def __init__(self) -> None: - """Create a new ``OpenFileTracker``.""" - self._descriptors: Dict[Any, OpenFile] = {} + ``fcntl`` locks are associated with an inode. If a process closes *any* file descriptor for an + inode, all fcntl locks the process holds on that inode are released, even if other descriptors + for the same inode are still open. - def get_fh(self, path: str) -> IO: - """Get a filehandle for a lockfile. + To avoid accidentally dropping locks we keep at most one open file descriptor per inode and + reference-count it. The descriptor is only closed when the reference count reaches zero (i.e. + no ``Lock`` in this process still needs it). - This routine will open writable files for read/write even if you're asking - for a shared (read-only) lock. This is so that we can upgrade to an exclusive - (write) lock later if requested. + Descriptors are *not* released on unlock; they are kept alive across lock/unlock cycles so that + the next lock operation can skip re-opening the file. ``Lock._ensure_valid_handle`` + re-validates the on-disk inode before each lock operation and drops a stale descriptor when + the file was deleted and replaced. + """ - Arguments: - path: path to lock file we want a filehandle for - """ - # Open writable files as rb+ so we can upgrade to write later - os_mode, fh_mode = (os.O_RDWR | os.O_CREAT), "rb+" + def __init__(self): + self._descriptors: Dict[DevIno, OpenFile] = {} - pid = os.getpid() - open_file = None # OpenFile object, if there is one - stat = None # stat result for the lockfile, if it exists + def get_ref_for_inode(self, key: DevIno) -> Optional[OpenFile]: + """Fast lookup: do we already have this inode open?""" + return self._descriptors.get(key) + def create_and_track(self, path: str) -> OpenFile: + """Slow path: Open file, handle directory creation, track it.""" + # Open the file and create it if it doesn't exist (incl. directories). try: - # see whether we've seen this inode/pid before - stat = os.stat(path) - key = (stat.st_dev, stat.st_ino, pid) - open_file = self._descriptors.get(key) - + try: + fd = os.open(path, os.O_RDWR | os.O_CREAT) + mode = "rb+" + except PermissionError: + fd = os.open(path, os.O_RDONLY) + mode = "rb" except OSError as e: - if e.errno != errno.ENOENT: # only handle file not found + if e.errno != errno.ENOENT: raise - - # path does not exist -- fail if we won't be able to create it - parent = os.path.dirname(path) or "." - if not os.access(parent, os.W_OK): + # Directory missing, create and retry + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + fd = os.open(path, os.O_RDWR | os.O_CREAT) + except OSError: raise CantCreateLockError(path) - - # if there was no already open file, we'll need to open one - if not open_file: - if stat and not os.access(path, os.W_OK): - # we know path exists but not if it's writable. If it's read-only, - # only open the file for reading (and fail if we're trying to get - # an exclusive (write) lock on it) - os_mode, fh_mode = os.O_RDONLY, "rb" - - fd = os.open(path, os_mode) - fh = os.fdopen(fd, fh_mode) - open_file = OpenFile(fh) - - # if we just created the file, we'll need to get its inode here - if not stat: - stat = os.fstat(fd) - key = (stat.st_dev, stat.st_ino, pid) - - self._descriptors[key] = open_file - - open_file.refs += 1 - return open_file.fh - - def release_by_stat(self, stat): - key = (stat.st_dev, stat.st_ino, os.getpid()) - open_file = self._descriptors.get(key) - assert open_file, "Attempted to close non-existing inode: %s" % stat.st_ino - + mode = "rb+" + + # Get file identifier (device, inode) for tracking. + stat = os.fstat(fd) + key = (stat.st_dev, stat.st_ino) + + # Did we open a file we already track, e.g. a symlink to existing tracker file. + if key in self._descriptors: + os.close(fd) + existing = self._descriptors[key] + existing.refs += 1 + return existing + + # Track the new file. + fh = os.fdopen(fd, mode) + obj = OpenFile(fh, key) + obj.refs += 1 + self._descriptors[key] = obj + return obj + + def release(self, open_file: OpenFile): + """Decrement the reference count and close the file handle when it reaches zero.""" open_file.refs -= 1 - if not open_file.refs: - del self._descriptors[key] + if open_file.refs <= 0: + if self._descriptors.get(open_file.key) is open_file: + del self._descriptors[open_file.key] open_file.fh.close() - def release_by_fh(self, fh): - self.release_by_stat(os.fstat(fh.fileno())) - def purge(self): - for key in list(self._descriptors.keys()): - self._descriptors[key].fh.close() - del self._descriptors[key] + """Close all tracked file descriptors and clear the cache.""" + for open_file in self._descriptors.values(): + open_file.fh.close() + self._descriptors.clear() #: Open file descriptors for locks in this process. Used to prevent one process @@ -198,16 +176,14 @@ def is_valid(op: int) -> bool: class Lock: """This is an implementation of a filesystem lock using Python's lockf. - In Python, ``lockf`` actually calls ``fcntl``, so this should work with - any filesystem implementation that supports locking through the fcntl - calls. This includes distributed filesystems like Lustre (when flock - is enabled) and recent NFS versions. + In Python, ``lockf`` actually calls ``fcntl``, so this should work with any filesystem + implementation that supports locking through the fcntl calls. This includes distributed + filesystems like Lustre (when flock is enabled) and recent NFS versions. - Note that this is for managing contention over resources *between* - processes and not for managing contention between threads in a process: the - functions of this object are not thread-safe. A process also must not - maintain multiple locks on the same file (or, more specifically, on - overlapping byte ranges in the same file). + Note that this is for managing contention over resources *between* processes and not for + managing contention between threads in a process: the functions of this object are not + thread-safe. A process also must not maintain multiple locks on the same file (or, more + specifically, on overlapping byte ranges in the same file). """ def __init__( @@ -222,29 +198,29 @@ def __init__( ) -> None: """Construct a new lock on the file at ``path``. - By default, the lock applies to the whole file. Optionally, - caller can specify a byte range beginning ``start`` bytes from - the start of the file and extending ``length`` bytes from there. + By default, the lock applies to the whole file. Optionally, caller can specify a byte + range beginning ``start`` bytes from the start of the file and extending ``length`` bytes + from there. - This exposes a subset of fcntl locking functionality. It does - not currently expose the ``whence`` parameter -- ``whence`` is - always ``os.SEEK_SET`` and ``start`` is always evaluated from the - beginning of the file. + This exposes a subset of fcntl locking functionality. It does not currently expose the + ``whence`` parameter -- ``whence`` is always ``os.SEEK_SET`` and ``start`` is always + evaluated from the beginning of the file. Args: path: path to the lock start: optional byte offset at which the lock starts length: optional number of bytes to lock - default_timeout: seconds to wait for lock attempts, - where None means to wait indefinitely + default_timeout: seconds to wait for lock attempts, where None means to wait + indefinitely debug: debug mode specific to locking - desc: optional debug message lock description, which is - helpful for distinguishing between different Spack locks. + desc: optional debug message lock description, which is helpful for distinguishing + between different Spack locks. """ self.path = path - self._file: Optional[IO[bytes]] = None self._reads = 0 self._writes = 0 + self._file_ref: Optional[OpenFile] = None + self._cached_key: Optional[DevIno] = None # byte range parameters self._start = start @@ -267,20 +243,68 @@ def __init__( self.host: Optional[str] = None self.old_host: Optional[str] = None + def _ensure_valid_handle(self) -> IO[bytes]: + """Return a valid file handle for the lock file, opening or re-opening as needed. + + On the happy path this costs a single ``os.stat`` syscall: if the inode on disk matches + ``_cached_key``, the already-open file handle is returned immediately. + + If the inode changed (the lock file was deleted and replaced by another process), the stale + reference is released and a fresh one is obtained. If the file does not exist yet it is + created (along with any missing parent directories). + """ + try: + # Check what is currently on disk. This is the only syscall in the happy path. + stat_res = os.stat(self.path) + current_key = (stat_res.st_dev, stat_res.st_ino) + + # Double-check that our cache corresponds the file on disk. + if self._file_ref and not self._file_ref.fh.closed: + if self._cached_key == current_key: + return self._file_ref.fh + + # Stale path: file was deleted and replaced on disk. + FILE_TRACKER.release(self._file_ref) + self._file_ref = None + + # Get reference to the verified inode from the tracker if it exist, or a new one. + existing_ref = FILE_TRACKER.get_ref_for_inode(current_key) + if existing_ref: + self._file_ref = existing_ref + self._file_ref.refs += 1 + else: + # We don't have it tracked, so we need to open and track it ourselves. + self._file_ref = FILE_TRACKER.create_and_track(self.path) + except OSError as e: + # Re-raise all errors except for "file not found". + if e.errno != errno.ENOENT: + raise + + # File was not found, so remove it from our cache. + if self._file_ref: + FILE_TRACKER.release(self._file_ref) + self._file_ref = None + + self._file_ref = FILE_TRACKER.create_and_track(self.path) + + # Update our local cache of what we hold + self._cached_key = self._file_ref.key + + return self._file_ref.fh + @staticmethod def _poll_interval_generator( _wait_times: Optional[Tuple[float, float, float]] = None, ) -> Generator[float, None, None]: - """This implements a backoff scheme for polling a contended resource - by suggesting a succession of wait times between polls. + """This implements a backoff scheme for polling a contended resource by suggesting a + succession of wait times between polls. - It suggests a poll interval of .1s until 2 seconds have passed, - then a poll interval of .2s until 10 seconds have passed, and finally - (for all requests after 10s) suggests a poll interval of .5s. + It suggests a poll interval of .1s until 2 seconds have passed, then a poll interval of + .2s until 10 seconds have passed, and finally (for all requests after 10s) suggests a poll + interval of .5s. - This doesn't actually track elapsed time, it estimates the waiting - time as though the caller always waits for the full length of time - suggested by this function. + This doesn't actually track elapsed time, it estimates the waiting time as though the + caller always waits for the full length of time suggested by this function. """ num_requests = 0 stage1, stage2, stage3 = _wait_times or (1e-1, 2e-1, 5e-1) @@ -295,27 +319,39 @@ def _poll_interval_generator( def __repr__(self) -> str: """Formal representation of the lock.""" - rep = "{0}(".format(self.__class__.__name__) + rep = f"{self.__class__.__name__}(" for attr, value in self.__dict__.items(): - rep += "{0}={1}, ".format(attr, value.__repr__()) - return "{0})".format(rep.strip(", ")) + rep += f"{attr}={value.__repr__()}, " + return f"{rep.strip(', ')})" def __str__(self) -> str: """Readable string (with key fields) of the lock.""" - location = "{0}[{1}:{2}]".format(self.path, self._start, self._length) - timeout = "timeout={0}".format(self.default_timeout) - activity = "#reads={0}, #writes={1}".format(self._reads, self._writes) - return "({0}, {1}, {2})".format(location, timeout, activity) + location = f"{self.path}[{self._start}:{self._length}]" + timeout = f"timeout={self.default_timeout}" + activity = f"#reads={self._reads}, #writes={self._writes}" + return f"({location}, {timeout}, {activity})" + + def __getstate__(self): + """Don't include file handles or counts in pickled state.""" + state = self.__dict__.copy() + del state["_file_ref"] + del state["_reads"] + del state["_writes"] + return state + + def __setstate__(self, state): + self.__dict__.update(state) + self._file_ref = None + self._reads = 0 + self._writes = 0 def _lock(self, op: int, timeout: Optional[float] = None) -> Tuple[float, int]: """This takes a lock using POSIX locks (``fcntl.lockf``). - The lock is implemented as a spin lock using a nonblocking call - to ``lockf()``. + The lock is implemented as a spin lock using a nonblocking call to ``lockf()``. - If the lock times out, it raises a ``LockError``. If the lock is - successfully acquired, the total wait time and the number of attempts - is returned. + If the lock times out, it raises a ``LockError``. If the lock is successfully acquired, the + total wait time and the number of attempts is returned. """ assert LockType.is_valid(op) op_str = LockType.to_str(op) @@ -323,12 +359,9 @@ def _lock(self, op: int, timeout: Optional[float] = None) -> Tuple[float, int]: self._log_acquiring("{0} LOCK".format(op_str)) timeout = timeout or self.default_timeout - # Create file and parent directories if they don't exist. - if self._file is None: - self._ensure_parent_directory() - self._file = FILE_TRACKER.get_fh(self.path) + fh = self._ensure_valid_handle() - if LockType.to_module(op) == fcntl.LOCK_EX and self._file.mode == "rb": + if LockType.to_module(op) == fcntl.LOCK_EX and fh.mode == "rb": # Attempt to upgrade to write lock w/a read-only file. # If the file were writable, we'd have opened it rb+ raise LockROFileError(self.path) @@ -352,25 +385,18 @@ def _lock(self, op: int, timeout: Optional[float] = None) -> Tuple[float, int]: time.sleep(next(poll_intervals)) num_attempts += 1 - raise LockTimeoutError( - op_str.lower(), self.path, time.monotonic() - start_time, num_attempts - ) + raise LockTimeoutError(op, self.path, time.monotonic() - start_time, num_attempts) def _poll_lock(self, op: int) -> bool: """Attempt to acquire the lock in a non-blocking manner. Return whether the locking attempt succeeds """ - assert self._file is not None, "cannot poll a lock without the file being set" + assert self._file_ref is not None, "cannot poll a lock without the file being set" + fh = self._file_ref.fh.fileno() module_op = LockType.to_module(op) try: # Try to get the lock (will raise if not available.) - fcntl.lockf( - self._file.fileno(), - module_op | fcntl.LOCK_NB, - self._length, - self._start, - os.SEEK_SET, - ) + fcntl.lockf(fh, module_op | fcntl.LOCK_NB, self._length, self._start, os.SEEK_SET) # help for debugging distributed locking if self.debug: @@ -395,22 +421,14 @@ def _poll_lock(self, op: int) -> bool: return False - def _ensure_parent_directory(self) -> str: - parent = os.path.dirname(self.path) - - # relative paths to lockfiles in the current directory have no parent - if not parent: - return "." - os.makedirs(parent, exist_ok=True) - return parent - def _read_log_debug_data(self) -> None: """Read PID and host data out of the file if it is there.""" - assert self._file is not None, "cannot read debug log without the file being set" + assert self._file_ref is not None, "cannot read debug log without the file being set" self.old_pid = self.pid self.old_host = self.host - line = self._file.read() + self._file_ref.fh.seek(0) + line = self._file_ref.fh.read() if line: pid, host = line.decode("utf-8").strip().split(",") _, _, pid = pid.rpartition("=") @@ -419,7 +437,7 @@ def _read_log_debug_data(self) -> None: def _write_log_debug_data(self) -> None: """Write PID and host data to the file, recording old values.""" - assert self._file is not None, "cannot write debug log without the file being set" + assert self._file_ref is not None, "cannot write debug log without the file being set" self.old_pid = self.pid self.old_host = self.host @@ -427,36 +445,33 @@ def _write_log_debug_data(self) -> None: self.host = socket.gethostname() # write pid, host to disk to sync over FS - self._file.seek(0) - self._file.write(f"pid={self.pid},host={self.host}".encode("utf-8")) - self._file.truncate() - self._file.flush() - os.fsync(self._file.fileno()) + self._file_ref.fh.seek(0) + self._file_ref.fh.write(f"pid={self.pid},host={self.host}".encode("utf-8")) + self._file_ref.fh.truncate() + self._file_ref.fh.flush() + os.fsync(self._file_ref.fh.fileno()) def _unlock(self) -> None: """Releases a lock using POSIX locks (``fcntl.lockf``) - Releases the lock regardless of mode. Note that read locks may - be masquerading as write locks, but this removes either. - + Releases the lock regardless of mode. Note that read locks may be masquerading as write + locks, but this removes either. """ - assert self._file is not None, "cannot unlock without the file being set" - fcntl.lockf(self._file.fileno(), fcntl.LOCK_UN, self._length, self._start, os.SEEK_SET) - FILE_TRACKER.release_by_fh(self._file) - self._file = None + assert self._file_ref is not None, "cannot unlock without the file being set" + fcntl.lockf( + self._file_ref.fh.fileno(), fcntl.LOCK_UN, self._length, self._start, os.SEEK_SET + ) self._reads = 0 self._writes = 0 def acquire_read(self, timeout: Optional[float] = None) -> bool: """Acquires a recursive, shared lock for reading. - Read and write locks can be acquired and released in arbitrary - order, but the POSIX lock is held until all local read and - write locks are released. - - Returns True if it is the first acquire and actually acquires - the POSIX lock, False if it is a nested transaction. + Read and write locks can be acquired and released in arbitrary order, but the POSIX lock is + held until all local read and write locks are released. + Returns True if it is the first acquire and actually acquires the POSIX lock, False if it + is a nested transaction. """ timeout = timeout or self.default_timeout @@ -469,19 +484,18 @@ def acquire_read(self, timeout: Optional[float] = None) -> bool: return True else: # Increment the read count for nested lock tracking + self._reaffirm_lock() self._reads += 1 return False def acquire_write(self, timeout: Optional[float] = None) -> bool: """Acquires a recursive, exclusive lock for writing. - Read and write locks can be acquired and released in arbitrary - order, but the POSIX lock is held until all local read and - write locks are released. - - Returns True if it is the first acquire and actually acquires - the POSIX lock, False if it is a nested transaction. + Read and write locks can be acquired and released in arbitrary order, but the POSIX lock + is held until all local read and write locks are released. + Returns True if it is the first acquire and actually acquires the POSIX lock, False if it + is a nested transaction. """ timeout = timeout or self.default_timeout @@ -499,15 +513,63 @@ def acquire_write(self, timeout: Optional[float] = None) -> bool: return self._reads == 0 else: # Increment the write count for nested lock tracking + self._reaffirm_lock() self._writes += 1 return False - def is_write_locked(self) -> bool: - """Check if the file is write locked + def _reaffirm_lock(self) -> None: + """Fork-safety: always re-affirm the lock with one non-blocking attempt. In the same + process, re-locking an already-held byte range succeeds instantly (POSIX). In a forked + child that doesn't own the POSIX lock, the call fails immediately and we raise. Use WRITE + if we hold an exclusive lock so we don't accidentally downgrade it.""" + if self._writes > 0: + op = LockType.WRITE + elif self._reads > 0: + op = LockType.READ + else: + return + self._ensure_valid_handle() + if not self._poll_lock(op): + raise LockTimeoutError(op, self.path, time=0, attempts=1) - Return: - (bool): ``True`` if the path is write locked, otherwise, ``False`` + def try_acquire_read(self) -> bool: + """Non-blocking attempt to acquire a shared read lock. + + Returns True if the lock was acquired, False if it would block. + """ + if self._reads == 0 and self._writes == 0: + self._ensure_valid_handle() + if not self._poll_lock(LockType.READ): + return False + self._reads += 1 + self._log_acquired("READ LOCK", 0, 1) + return True + else: + self._reaffirm_lock() + self._reads += 1 + return True + + def try_acquire_write(self) -> bool: + """Non-blocking attempt to acquire an exclusive write lock. + + Returns True if the lock was acquired, False if it would block. """ + if self._writes == 0: + fh = self._ensure_valid_handle() + if LockType.to_module(LockType.WRITE) == fcntl.LOCK_EX and fh.mode == "rb": + raise LockROFileError(self.path) + if not self._poll_lock(LockType.WRITE): + return False + self._writes += 1 + self._log_acquired("WRITE LOCK", 0, 1) + return True + else: + self._reaffirm_lock() + self._writes += 1 + return True + + def is_write_locked(self) -> bool: + """Returns ``True`` if the path is write locked, otherwise, ``False``""" try: self.acquire_read() @@ -520,8 +582,7 @@ def is_write_locked(self) -> bool: return False def downgrade_write_to_read(self, timeout: Optional[float] = None) -> None: - """ - Downgrade from an exclusive write lock to a shared read. + """Downgrade from an exclusive write lock to a shared read. Raises: LockDowngradeError: if this is an attempt at a nested transaction @@ -539,8 +600,7 @@ def downgrade_write_to_read(self, timeout: Optional[float] = None) -> None: raise LockDowngradeError(self.path) def upgrade_read_to_write(self, timeout: Optional[float] = None) -> None: - """ - Attempts to upgrade from a shared read lock to an exclusive write. + """Attempts to upgrade from a shared read lock to an exclusive write. Raises: LockUpgradeError: if this is an attempt at a nested transaction @@ -561,19 +621,17 @@ def release_read(self, release_fn: ReleaseFnType = None) -> bool: """Releases a read lock. Arguments: - release_fn (typing.Callable): function to call *before* the last recursive - lock (read or write) is released. - - If the last recursive lock will be released, then this will call - release_fn and return its result (if provided), or return True - (if release_fn was not provided). + release_fn: function to call *before* the last recursive lock (read or write) is + released. - Otherwise, we are still nested inside some other lock, so do not - call the release_fn and, return False. + If the last recursive lock will be released, then this will call release_fn and return its + result (if provided), or return True (if release_fn was not provided). - Does limited correctness checking: if a read lock is released - when none are held, this will raise an assertion error. + Otherwise, we are still nested inside some other lock, so do not call the release_fn and, + return False. + Does limited correctness checking: if a read lock is released when none are held, this + will raise an assertion error. """ assert self._reads > 0 @@ -588,7 +646,7 @@ def release_read(self, release_fn: ReleaseFnType = None) -> bool: self._unlock() # can raise LockError. self._reads = 0 self._log_released(locktype) - return result + return bool(result) else: self._reads -= 1 return False @@ -597,42 +655,37 @@ def release_write(self, release_fn: ReleaseFnType = None) -> bool: """Releases a write lock. Arguments: - release_fn (typing.Callable): function to call before the last recursive - write is released. + release_fn: function to call before the last recursive write is released. - If the last recursive *write* lock will be released, then this - will call release_fn and return its result (if provided), or - return True (if release_fn was not provided). Otherwise, we are - still nested inside some other write lock, so do not call the - release_fn, and return False. - - Does limited correctness checking: if a read lock is released - when none are held, this will raise an assertion error. + If the last recursive *write* lock will be released, then this will call release_fn and + return its result (if provided), or return True (if release_fn was not provided). + Otherwise, we are still nested inside some other write lock, so do not call the release_fn, + and return False. + Does limited correctness checking: if a read lock is released when none are held, this + will raise an assertion error. """ assert self._writes > 0 release_fn = release_fn or true_fn locktype = "WRITE LOCK" - if self._writes == 1 and self._reads == 0: + if self._writes == 1: self._log_releasing(locktype) # we need to call release_fn before releasing the lock result = release_fn() - self._unlock() # can raise LockError. + if self._reads > 0: + self._lock(LockType.READ) + else: + self._unlock() # can raise LockError. + self._writes = 0 self._log_released(locktype) - return result + return bool(result) else: self._writes -= 1 - - # when the last *write* is released, we call release_fn here - # instead of immediately before releasing the lock. - if self._writes == 0: - return release_fn() - else: - return False + return False def cleanup(self) -> None: if self._reads == 0 and self._writes == 0: @@ -697,46 +750,27 @@ class LockTransaction: Arguments: lock: underlying lock for this transaction to be acquired on enter and released on exit - acquire: function to be called after lock is acquired, or contextmanager to enter after - acquire and leave before release. - release: function to be called before release. If ``acquire`` is a contextmanager, this - will be called *after* exiting the nested context and before the lock is released. + acquire: function to be called after lock is acquired + release: function to be called before release, with ``(exc_type, exc_value, traceback)`` timeout: number of seconds to set for the timeout when acquiring the lock (default no timeout) - - If the ``acquire_fn`` returns a value, it is used as the return value for - ``__enter__``, allowing it to be passed as the ``as`` argument of a - ``with`` statement. - - If ``acquire_fn`` returns a context manager, *its* ``__enter__`` function - will be called after the lock is acquired, and its ``__exit__`` function - will be called before ``release_fn`` in ``__exit__``, allowing you to - nest a context manager inside this one. - - Timeout for lock is customizable. - """ def __init__( self, lock: Lock, - acquire: Union[ReleaseFnType, ContextManager] = None, - release: Union[ReleaseFnType, ContextManager] = None, + acquire: Optional[Callable[[], None]] = None, + release: Optional[ExitFnType] = None, timeout: Optional[float] = None, ) -> None: self._lock = lock self._timeout = timeout self._acquire_fn = acquire self._release_fn = release - self._as = None def __enter__(self): if self._enter() and self._acquire_fn: - self._as = self._acquire_fn() - if hasattr(self._as, "__enter__"): - return self._as.__enter__() - else: - return self._as + return self._acquire_fn() def __exit__( self, @@ -744,26 +778,17 @@ def __exit__( exc_value: Optional[BaseException], traceback: Optional[TracebackType], ) -> bool: - suppress = False - def release_fn(): if self._release_fn is not None: return self._release_fn(exc_type, exc_value, traceback) - if self._as and hasattr(self._as, "__exit__"): - if self._as.__exit__(exc_type, exc_value, traceback): - suppress = True - - if self._exit(release_fn): - suppress = True - - return suppress + return bool(self._exit(release_fn)) def _enter(self) -> bool: - return NotImplemented + raise NotImplementedError def _exit(self, release_fn: ReleaseFnType) -> bool: - return NotImplemented + raise NotImplementedError class ReadTransaction(LockTransaction): @@ -793,7 +818,7 @@ class LockError(Exception): class LockDowngradeError(LockError): """Raised when unable to downgrade from a write to a read lock.""" - def __init__(self, path): + def __init__(self, path: str) -> None: msg = "Cannot downgrade lock from write to read on file: %s" % path super().__init__(msg) @@ -801,11 +826,12 @@ def __init__(self, path): class LockTimeoutError(LockError): """Raised when an attempt to acquire a lock times out.""" - def __init__(self, lock_type, path, time, attempts): + def __init__(self, lock_type: int, path: str, time: float, attempts: int) -> None: + lock_type_str = LockType.to_str(lock_type).lower() fmt = "Timed out waiting for a {} lock after {}.\n Made {} {} on file: {}" super().__init__( fmt.format( - lock_type, + lock_type_str, lang.pretty_seconds(time), attempts, "attempt" if attempts == 1 else "attempts", @@ -817,7 +843,7 @@ def __init__(self, lock_type, path, time, attempts): class LockUpgradeError(LockError): """Raised when unable to upgrade from a read to a write lock.""" - def __init__(self, path): + def __init__(self, path: str) -> None: msg = "Cannot upgrade lock from read to write on file: %s" % path super().__init__(msg) @@ -829,7 +855,7 @@ class LockPermissionError(LockError): class LockROFileError(LockPermissionError): """Tried to take an exclusive lock on a read-only file.""" - def __init__(self, path): + def __init__(self, path: str) -> None: msg = "Can't take write lock on read-only file: %s" % path super().__init__(msg) @@ -837,7 +863,7 @@ def __init__(self, path): class CantCreateLockError(LockPermissionError): """Attempt to create a lock in an unwritable location.""" - def __init__(self, path): + def __init__(self, path: str) -> None: msg = "cannot create lock '%s': " % path msg += "file does not exist and location is not writable" super().__init__(msg) diff --git a/lib/spack/spack/llnl/util/tty/__init__.py b/lib/spack/spack/llnl/util/tty/__init__.py index 775a040c56b0fe..80a9ea0ab48f90 100644 --- a/lib/spack/spack/llnl/util/tty/__init__.py +++ b/lib/spack/spack/llnl/util/tty/__init__.py @@ -11,7 +11,7 @@ import traceback from datetime import datetime from types import TracebackType -from typing import Callable, Iterator, NoReturn, Optional, Type, Union +from typing import IO, Callable, Iterator, NoReturn, Optional, Type, Union from .color import cescape, clen, cprint, cwrite @@ -185,7 +185,7 @@ def info( message: Union[Exception, str], *args, format: str = "*b", - stream: Optional[io.IOBase] = None, + stream: Optional[IO[str]] = None, wrap: bool = False, break_long_words: bool = False, countback: int = 3, @@ -201,7 +201,7 @@ def info( cprint( "@%s{%s==>} %s%s" % (format, st_text, get_timestamp(), cescape(_output_filter(str(message)))), - stream=stream, # type: ignore[arg-type] + stream=stream, ) for arg in args: if wrap: @@ -225,44 +225,30 @@ def verbose(message, *args, format: str = "c", **kwargs) -> None: def debug( - message, *args, level: int = 1, format: str = "g", stream: Optional[io.IOBase] = None, **kwargs + message, *args, level: int = 1, format: str = "g", stream: Optional[IO[str]] = None, **kwargs ) -> None: """Print a debug message if the debug level is set.""" if is_debug(level): stream_arg = stream or sys.stderr - info(message, *args, format=format, stream=stream_arg, **kwargs) # type: ignore[arg-type] + info(message, *args, format=format, stream=stream_arg, **kwargs) -def error( - message, *args, format: str = "*r", stream: Optional[io.IOBase] = None, **kwargs -) -> None: +def error(message, *args, format: str = "*r", stream: Optional[IO[str]] = None, **kwargs) -> None: """Print an error message.""" if not error_enabled(): return stream = stream or sys.stderr - info( - f"Error: {message}", - *args, - format=format, - stream=stream, # type: ignore[arg-type] - **kwargs, - ) + info(f"Error: {message}", *args, format=format, stream=stream, **kwargs) -def warn(message, *args, format: str = "*Y", stream: Optional[io.IOBase] = None, **kwargs) -> None: +def warn(message, *args, format: str = "*Y", stream: Optional[IO[str]] = None, **kwargs) -> None: """Print a warning message.""" if not warn_enabled(): return stream = stream or sys.stderr - info( - f"Warning: {message}", - *args, - format=format, - stream=stream, # type: ignore[arg-type] - **kwargs, - ) + info(f"Warning: {message}", *args, format=format, stream=stream, **kwargs) def die(message, *args, countback: int = 4, **kwargs) -> NoReturn: diff --git a/lib/spack/spack/llnl/util/tty/colify.py b/lib/spack/spack/llnl/util/tty/colify.py index beeb535887b0f0..d6b7a8cfdd5712 100644 --- a/lib/spack/spack/llnl/util/tty/colify.py +++ b/lib/spack/spack/llnl/util/tty/colify.py @@ -5,6 +5,7 @@ """ Routines for printing columnar output. See ``colify()`` for more information. """ + import io import os import shutil diff --git a/lib/spack/spack/llnl/util/tty/color.py b/lib/spack/spack/llnl/util/tty/color.py index 290ca8d70c2cbc..caa327bbc28fa1 100644 --- a/lib/spack/spack/llnl/util/tty/color.py +++ b/lib/spack/spack/llnl/util/tty/color.py @@ -58,13 +58,14 @@ To output an ``@``, use ``@@``. To output a ``}`` inside braces, use ``}}``. """ + import io import os import re import sys import textwrap from contextlib import contextmanager -from typing import Iterator, List, NamedTuple, Optional, Tuple, Union +from typing import IO, Iterator, List, NamedTuple, Optional, Tuple, Union class ColorParseError(Exception): @@ -185,11 +186,13 @@ def _err_check(result, func, args): _force_color = False -def get_color_when() -> bool: +def get_color_when(stdout=None) -> bool: """Return whether commands should print color or not.""" if _force_color is not None: return _force_color - return sys.stdout.isatty() + if stdout is None: + stdout = sys.stdout + return stdout.isatty() def set_color_when(when: Union[str, bool, None]) -> None: @@ -265,6 +268,9 @@ def match_to_ansi(match) -> str: semi = ";" if color_number else "" ansi_code = _escape(f"{styles[style]}{semi}{color_number}", color, enclose, zsh) if text: + # must be here, not in the final return: top-level @@ is already handled by + # the regex, and its @-results could form new @@ pairs. + text = text.replace("@@", "@") return f"{ansi_code}{text}{_escape(0, color, enclose, zsh)}" else: return ansi_code @@ -299,7 +305,7 @@ def plain_to_color(self, index: int) -> int: def cmapping(string: str) -> ColorMapping: """Return a mapping for translating indices in a plain string to indices in colored text. - The returned dictionary maps indices in the plain string to the offset of the cooresponding + The returned dictionary maps indices in the plain string to the offset of the corresponding indices in the colored string. """ @@ -369,7 +375,7 @@ def cextra(string: str) -> int: return len("".join(re.findall(r"\033[^m]*m", string))) -def cwrite(string: str, stream: Optional[io.IOBase] = None, color: Optional[bool] = None) -> None: +def cwrite(string: str, stream: Optional[IO[str]] = None, color: Optional[bool] = None) -> None: """Replace all color expressions in string with ANSI control codes and write the result to the stream. If color is False, this will write plain text with no color. If True, @@ -382,7 +388,7 @@ def cwrite(string: str, stream: Optional[io.IOBase] = None, color: Optional[bool stream.write(colorize(string, color=color)) -def cprint(string: str, stream: Optional[io.IOBase] = None, color: Optional[bool] = None) -> None: +def cprint(string: str, stream: Optional[IO[str]] = None, color: Optional[bool] = None) -> None: """Same as cwrite, but writes a trailing newline to the stream.""" cwrite(string + "\n", stream, color) diff --git a/lib/spack/spack/llnl/util/tty/log.py b/lib/spack/spack/llnl/util/tty/log.py index 5072624e5e305f..fba3eff45bb849 100644 --- a/lib/spack/spack/llnl/util/tty/log.py +++ b/lib/spack/spack/llnl/util/tty/log.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Utility classes for logging the output of blocks of code.""" + import atexit import ctypes import errno @@ -13,20 +14,27 @@ import select import signal import sys -import threading import traceback from contextlib import contextmanager from multiprocessing.connection import Connection from threading import Thread -from typing import IO, Callable, Optional, Tuple +from typing import IO, Callable, List, Optional, Tuple import spack.llnl.util.tty as tty +if sys.platform == "win32": + import ctypes.wintypes as wintypes + import msvcrt + + kernel32 = ctypes.windll.kernel32 + try: import termios except ImportError: termios = None # type: ignore[assignment] +# win32api constants +DUPLICATE_SAME_ACCESS = 0x00000002 esc, bell, lbracket, bslash, newline = r"\x1b", r"\x07", r"\[", r"\\", r"\n" # Ansi Control Sequence Introducers (CSI) are a well-defined format @@ -158,7 +166,7 @@ class keyboard_input(preserve_terminal_settings): [Running] <------- bg sends SIGCONT ---------- [Stopped] [ in BG ] [ in BG ] - We handle all transitions exept for ``SIGTSTP`` generated by Ctrl-Z + We handle all transitions except for ``SIGTSTP`` generated by Ctrl-Z by periodically calling ``check_fg_bg()``. This routine notices if we are in the background with canonical mode or echo disabled, or if we are in the foreground without canonical disabled and echo enabled, @@ -547,67 +555,78 @@ class StreamWrapper: def __init__(self, sys_attr): self.sys_attr = sys_attr self.saved_stream = None - if sys.platform.startswith("win32"): - if hasattr(sys, "gettotalrefcount"): # debug build - libc = ctypes.CDLL("ucrtbased") - else: - libc = ctypes.CDLL("api-ms-win-crt-stdio-l1-1-0") - - kernel32 = ctypes.WinDLL("kernel32") - - # https://docs.microsoft.com/en-us/windows/console/getstdhandle - if self.sys_attr == "stdout": - STD_HANDLE = -11 - elif self.sys_attr == "stderr": - STD_HANDLE = -12 - else: - raise KeyError(self.sys_attr) - - c_stdout = kernel32.GetStdHandle(STD_HANDLE) - self.libc = libc - self.c_stream = c_stdout + + kernel32.SetStdHandle.argtypes = [wintypes.DWORD, wintypes.HANDLE] # nStdHandle # hHandle + + kernel32.GetStdHandle.argtypes = [wintypes.DWORD] + kernel32.GetStdHandle.restype = wintypes.HANDLE + + # https://docs.microsoft.com/en-us/windows/console/getstdhandle + if self.sys_attr == "stdout": + self.STD_HANDLE = -11 + elif self.sys_attr == "stderr": + self.STD_HANDLE = -12 else: - self.libc = ctypes.CDLL(None) - self.c_stream = ctypes.c_void_p.in_dll(self.libc, self.sys_attr) - self.sys_stream = getattr(sys, self.sys_attr) - self.orig_stream_fd = self.sys_stream.fileno() - # Save a copy of the original stdout fd in saved_stream - self.saved_stream = os.dup(self.orig_stream_fd) - - def redirect_stream(self, to_fd): + raise KeyError(self.sys_attr) + + self.saved_stream = getattr(sys, self.sys_attr) + self.std_fd = self.saved_stream.fileno() + self.saved_std_handle = kernel32.GetStdHandle(self.STD_HANDLE) + self.saved_stream_fd = os.dup(self.std_fd) + self.redirect_fd = None + + def redirect_stream(self, write_conn): """Redirect stdout to the given file descriptor.""" - # Flush the C-level buffer stream - if sys.platform.startswith("win32"): - self.libc.fflush(None) - else: - self.libc.fflush(self.c_stream) - # Flush and close sys_stream - also closes the file descriptor (fd) - sys_stream = getattr(sys, self.sys_attr) - sys_stream.flush() - sys_stream.close() - # Make orig_stream_fd point to the same file as to_fd - os.dup2(to_fd, self.orig_stream_fd) - # Set sys_stream to a new stream that points to the redirected fd - new_buffer = open(self.orig_stream_fd, "wb") - new_stream = io.TextIOWrapper(new_buffer) - setattr(sys, self.sys_attr, new_stream) - self.sys_stream = getattr(sys, self.sys_attr) + self.flush() + # Get fd for new stream + redirect_h = write_conn.fileno() + dup_redirect_h = dup_fh(redirect_h) + os.set_handle_inheritable(redirect_h, True) + self.redirect_fd = msvcrt.open_osfhandle(dup_redirect_h, os.O_WRONLY) + kernel32.SetStdHandle(self.STD_HANDLE, wintypes.HANDLE(redirect_h)) + os.dup2(self.redirect_fd, self.std_fd) + setattr( + sys, + self.sys_attr, + os.fdopen( + self.std_fd, + "w", + encoding="utf-8", + buffering=1, + errors="replace", + closefd=False, + newline="\n", + ), + ) def flush(self): - if sys.platform.startswith("win32"): - self.libc.fflush(None) - else: - self.libc.fflush(self.c_stream) - self.sys_stream.flush() + # get current system stream for the standard fd we're redirecting + sys_stream = getattr(sys, self.sys_attr) + try: + if sys_stream: + # Flush the system stream before redirection + sys_stream.flush() + except BaseException as e: + # swallow flush errors + tty.debug(f"Encountered error flushing stream: {e}") + pass def close(self): """Redirect back to the original system stream, and close stream""" try: - if self.saved_stream is not None: - self.redirect_stream(self.saved_stream) + self.flush() + if self.saved_stream_fd is not None: + # restore os handle + kernel32.SetStdHandle(self.STD_HANDLE, self.saved_std_handle) + # restore c fd + os.dup2(self.saved_stream_fd, self.std_fd) + # python level + setattr(sys, self.sys_attr, self.saved_stream) finally: - if self.saved_stream is not None: - os.close(self.saved_stream) + if self.redirect_fd is not None: + os.close(self.redirect_fd) + if self.saved_stream_fd is not None: + os.close(self.saved_stream_fd) class winlog: @@ -630,60 +649,37 @@ def __init__( self.old_stdout = sys.stdout self.old_stderr = sys.stderr self.append = append + self.filter_fn = filter_fn + self.read_p, self.write_p = None, None + self._thread = None def __enter__(self): if self._active: raise RuntimeError("Can't re-enter the same log_output!") - # Open both write and reading on logfile - write_mode = "ab+" if self.append else "wb+" - self.writer = open(self.logfile, mode=write_mode) - self.reader = open(self.logfile, mode="rb+") + self.read_p, self.write_p = multiprocessing.Pipe(duplex=False) # Dup stdout so we can still write to it after redirection - self.echo_writer = open(os.dup(sys.stdout.fileno()), "w", encoding=sys.stdout.encoding) - # Redirect stdout and stderr to write to logfile - self.stderr.redirect_stream(self.writer.fileno()) - self.stdout.redirect_stream(self.writer.fileno()) - self._kill = threading.Event() - - def background_reader(reader, echo_writer, _kill): - # for each line printed to logfile, read it - # if echo: write line to user - try: - while True: - is_killed = _kill.wait(0.1) - # Flush buffered build output to file - # stdout/err fds refer to log file - self.stderr.flush() - self.stdout.flush() - - line = reader.readline() - if self.echo and line: - echo_writer.write("{0}".format(line.decode())) - echo_writer.flush() - - if is_killed: - break - finally: - reader.close() + original_stdout_fd = sys.stdout.fileno() + echo_writer = os.fdopen(os.dup(original_stdout_fd), "w", encoding="utf-8", newline="\n") self._active = True self._thread = Thread( - target=background_reader, args=(self.reader, self.echo_writer, self._kill) + target=self._background_reader, + args=(self.read_p, self.logfile, echo_writer, self.append, self.echo, self.filter_fn), ) self._thread.start() + # Redirect stdout and stderr to write to logfile + self.stderr.redirect_stream(self.write_p) + self.stdout.redirect_stream(self.write_p) + return self def __exit__(self, exc_type, exc_val, exc_tb): - self.writer.close() - self.echo_writer.flush() - self.stdout.flush() - self.stderr.flush() - self._kill.set() - self._thread.join() self.stdout.close() self.stderr.close() + self.write_p.close() + self._thread.join() self._active = False @contextmanager @@ -691,7 +687,65 @@ def force_echo(self): """Context manager to force local echo, even if echo is off.""" if not self._active: raise RuntimeError("Can't call force_echo() outside log_output region!") - yield + sys.stdout.write(xon) + sys.stdout.flush() + try: + yield + finally: + sys.stdout.write(xoff) + sys.stdout.flush() + + @staticmethod + def _background_reader( + read, + logfile: str, + stdout: io.TextIOWrapper, + append: bool, + echo: bool, + filter_fn: Optional[Callable], + ): + force_echo = False + + write_mode = "ab" if append else "wb" + log_writer = open(logfile, mode=write_mode) + try: + while True: + data = read.recv_bytes(maxlength=4096) + if not data: + # the pipe is closed or otherwise inaccesible + return + norm_data = data.decode(encoding="utf-8", errors="replace") + clean_line, num_controls = control.subn("", norm_data) + + log_writer.write(_strip(clean_line).encode(encoding="utf-8")) + log_writer.flush() + if echo or force_echo: + output = clean_line + if filter_fn: + output = filter_fn(output) + enc = stdout.encoding + if enc != "utf-8": + output = output.encode(enc, "replace").decode(enc) + stdout.write(output) + stdout.flush() + if num_controls > 0: + controls = control.findall(norm_data) + force_echo = force_echo_on(force_echo, controls) + if read.closed: + break + + # swallow valid errors + except EOFError: + pass + except OSError: + pass + except BaseException as e: + tty.error(f"Exception in log writer thread! {e}", stream=stdout) + traceback.print_exc(file=stdout) + finally: + read.close() + log_writer.close() + stdout.close() def _writer_daemon( @@ -835,10 +889,7 @@ def _writer_daemon( if num_controls > 0: controls = control.findall(line) - if xon in controls: - force_echo = True - if xoff in controls: - force_echo = False + force_echo = force_echo_on(force_echo, controls) if not _input_available(read_file): break @@ -862,5 +913,59 @@ def _writer_daemon( control_fd.send(echo) +if sys.platform == "win32": + # dont define this outside windows, otherwise mypy complains + # or we'd have to # type: ignore on basically every line of + # this method + def dup_fh(fh: int) -> int: + """Windows Only + Duplicates Windows file handles. Useful when + we need multiple references to a single file handle + that all can be closed independently + + uses DuplicateHandle from the win32 api + + Arguments: + fh: OS level file handle to be duplicated + + Returns: integer representing the new, identical file handle + """ + # Define function signatures for safety + kernel32.DuplicateHandle.argtypes = [ + wintypes.HANDLE, # hSourceProcessHandle + wintypes.HANDLE, # hSourceHandle + wintypes.HANDLE, # hTargetProcessHandle + ctypes.POINTER(wintypes.HANDLE), # lpTargetHandle + wintypes.DWORD, # dwDesiredAccess + wintypes.BOOL, # bInheritHandle + wintypes.DWORD, # dwOptions + ] + current_process = kernel32.GetCurrentProcess() + target_handle = wintypes.HANDLE() + + success = kernel32.DuplicateHandle( + current_process, + wintypes.HANDLE(fh), + current_process, + ctypes.byref(target_handle), + 0, + True, + DUPLICATE_SAME_ACCESS, + ) + + if not success or not target_handle.value: + raise ctypes.WinError() + + return target_handle.value + + +def force_echo_on(force_echo: bool, controls: List[str]): + if xon in controls: + return True + if xoff in controls: + return False + return force_echo + + def _input_available(f): return f in select.select([f], [], [], 0)[0] diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 5ec3d8fc574a59..cb634fedf0f399 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -7,8 +7,11 @@ In a normal Spack installation, this is invoked from the bin/spack script after the system path is set up. """ + import argparse +import gc import inspect +import multiprocessing import operator import os import pstats @@ -40,8 +43,6 @@ import spack.platforms import spack.solver.asp import spack.spec -import spack.store -import spack.util.debug import spack.util.environment import spack.util.lock @@ -155,7 +156,6 @@ def _format_usage(self, usage, actions, groups, prefix=None): def add_argument(self, action): if action.help is not argparse.SUPPRESS: - # find all invocations get_invocation = self._format_action_invocation invocation_lengths = [color.clen(get_invocation(action)) + self._current_indent] @@ -369,6 +369,7 @@ def add_command(self, cmd_name): help=module.description, description=module.description, ) + subparser.set_defaults(subparser=subparser) module.setup_parser(subparser) # return the callable function for the command @@ -504,7 +505,7 @@ def make_argument_parser(**kwargs): default="SPACK_BACKTRACE" in os.environ, help="always show backtraces for exceptions", ) - debug.add_argument("--pdb", action="store_true", help="run spack under the pdb debugger") + debug.add_argument("--pdb", action="store_true", help=argparse.SUPPRESS) debug.add_argument("--timestamp", action="store_true", help="add a timestamp to tty output") debug.add_argument( "-m", "--mock", action="store_true", help="use mock packages instead of real ones" @@ -537,28 +538,12 @@ def make_argument_parser(**kwargs): help="do not use filesystem locking (unsafe)", ) - profile = parser.add_argument_group("profiling") - profile.add_argument( - "-p", - "--profile", - action="store_true", - dest="spack_profile", - help="profile execution using cProfile", - ) - profile.add_argument("--profile-file", default=None, help="Filename to save profile data to.") - profile.add_argument( - "--sorted-profile", - default=None, - metavar="STAT", - help="profile and sort by STAT, which can be: calls, ncalls,\n" - "cumtime, cumulative, filename, line, module", - ) - profile.add_argument( - "--lines", - default=20, - action="store", - help="lines of profile output or 'all' (default: 20)", + debug.add_argument( + "-p", "--profile", action="store_true", dest="spack_profile", help=argparse.SUPPRESS ) + debug.add_argument("--profile-file", default=None, help=argparse.SUPPRESS) + debug.add_argument("--sorted-profile", default=None, metavar="STAT", help=argparse.SUPPRESS) + debug.add_argument("--lines", default=20, action="store", help=argparse.SUPPRESS) return parser @@ -586,7 +571,6 @@ def setup_main_options(args): spack.error.SHOW_BACKTRACE = True if args.debug: - spack.util.debug.register_interrupt_handler() spack.config.set("config:debug", True, scope="command_line") spack.util.environment.TRACING_ENABLED = True @@ -645,7 +629,7 @@ def _invoke_command(command, parser, args, unknown_args): return_val = command(parser, args, unknown_args) else: if unknown_args: - tty.die("unrecognized arguments: %s" % " ".join(unknown_args)) + args.subparser.error("unrecognized arguments: %s" % " ".join(unknown_args)) return_val = command(parser, args) # Allow commands to return and error code if they want @@ -847,17 +831,6 @@ def shell_set(var, value): roots_val = ":".join(reversed(paths)) shell_set("_sp_%s_roots" % name, roots_val) - # print environment module system if available. This can be expensive - # on clusters, so skip it if not needed. - if "modules" in info: - generic_arch = spack.vendor.archspec.cpu.host().family - module_spec = "environment-modules target={0}".format(generic_arch) - specs = spack.store.STORE.db.query(module_spec) - if specs: - shell_set("_sp_module_prefix", specs[-1].prefix) - else: - shell_set("_sp_module_prefix", "not_installed") - def restore_macos_dyld_vars(): """ @@ -898,8 +871,7 @@ def resolve_alias(cmd_name: str, cmd: List[str]) -> Tuple[str, List[str]]: ) if key in all_commands: tty.warn( - f"Alias '{key}' (mapping to '{value}') attempts to override" - " built-in command." + f"Alias '{key}' (mapping to '{value}') attempts to override built-in command." ) if cmd_name not in all_commands: @@ -979,7 +951,7 @@ def _main(argv=None): # them, which reduces startup latency. parser = make_argument_parser() parser.add_argument("command", nargs=argparse.REMAINDER) - args, unknown = parser.parse_known_args(argv) + args = parser.parse_args(argv) # Just print help and exit if run with no arguments at all no_args = (len(sys.argv) == 1) if argv is None else (len(argv) == 0) @@ -1096,8 +1068,39 @@ def finish_parse_and_run(parser, cmd_name, main_args, env_format_error): # now we can actually execute the command. if main_args.spack_profile or main_args.sorted_profile or main_args.profile_file: + new_args = [sys.executable, "-m", "cProfile"] + if main_args.sorted_profile: + new_args.extend(["-s", main_args.sorted_profile]) + if main_args.profile_file: + new_args.extend(["-o", main_args.profile_file]) + new_args.append(spack.paths.spack_script) + skip_next = False + for arg in sys.argv[1:]: + if skip_next: + skip_next = False + continue + if arg in ("--sorted-profile", "--profile-file", "--lines"): + skip_next = True + continue + if arg.startswith(("--sorted-profile=", "--profile-file=", "--lines=")): + continue + if arg in ("--profile", "-p"): + continue + new_args.append(arg) + formatted_args = " ".join(shlex.quote(a) for a in new_args) + tty.warn( + "The --profile flag is deprecated and will be removed in Spack v1.3. " + f"Use `{formatted_args}` instead." + ) _profile_wrapper(command, main_args, parser, args, unknown) elif main_args.pdb: + new_args = [sys.executable, "-m", "pdb", spack.paths.spack_script] + new_args.extend(arg for arg in sys.argv[1:] if arg != "--pdb") + formatted_args = " ".join(shlex.quote(arg) for arg in new_args) + tty.warn( + "The --pdb flag is deprecated and will be removed in Spack v1.3. " + f"Use `{formatted_args}` instead." + ) import pdb pdb.runctx("_invoke_command(command, parser, args, unknown)", globals(), locals()) @@ -1119,7 +1122,12 @@ def main(argv=None): the executable name. If None, parses from sys.argv. """ + # When using the forkserver start method, preload the following modules to improve startup + # time of child processes. + multiprocessing.set_forkserver_preload(["spack.main", "spack.package", "spack.new_installer"]) try: + g0, g1, g2 = gc.get_threshold() + gc.set_threshold(50 * g0, g1, g2) return _main(argv) except spack.solver.asp.OutputDoesNotSatisfyInputError as e: diff --git a/lib/spack/spack/mirrors/mirror.py b/lib/spack/spack/mirrors/mirror.py index 3ace9ec4743156..0a0fb8717e3b00 100644 --- a/lib/spack/spack/mirrors/mirror.py +++ b/lib/spack/spack/mirrors/mirror.py @@ -4,12 +4,11 @@ import operator import os import urllib.parse -from typing import Any, Dict, List, Mapping, Optional, Tuple, Union +from typing import IO, Any, Dict, Iterator, List, Mapping, Optional, Tuple, Union, overload import spack.config import spack.llnl.util.tty as tty import spack.util.path -import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml import spack.util.url as url_util from spack.error import MirrorError @@ -44,27 +43,20 @@ class Mirror: to them. These two URLs are usually the same. """ - def __init__(self, data: Union[str, dict], name: Optional[str] = None): + def __init__(self, data: Union[str, dict], name: Optional[str] = None) -> None: self._data = data self._name = name @staticmethod - def from_yaml(stream, name=None): + def from_yaml(stream: Union[str, IO[str]], name: Optional[str] = None) -> "Mirror": return Mirror(syaml.load(stream), name) @staticmethod - def from_json(stream, name=None): - try: - return Mirror(sjson.load(stream), name) - except Exception as e: - raise sjson.SpackJSONError("error parsing JSON mirror:", str(e)) from e - - @staticmethod - def from_local_path(path: str): + def from_local_path(path: str) -> "Mirror": return Mirror(url_util.path_to_file_url(path)) @staticmethod - def from_url(url: str): + def from_url(url: str) -> "Mirror": """Create an anonymous mirror by URL. This method validates the URL.""" if urllib.parse.urlparse(url).scheme not in supported_url_schemes: raise ValueError( @@ -73,27 +65,31 @@ def from_url(url: str): ) return Mirror(url) - def __eq__(self, other): + def __eq__(self, other: object) -> bool: if not isinstance(other, Mirror): return NotImplemented return self._data == other._data and self._name == other._name - def __str__(self): + def __str__(self) -> str: return f"{self._name}: {self.push_url} {self.fetch_url}" - def __repr__(self): + def __repr__(self) -> str: return f"Mirror(name={self._name!r}, data={self._data!r})" - def to_json(self, stream=None): - return sjson.dump(self.to_dict(), stream) + @overload + def to_yaml(self, stream: None = ...) -> str: ... - def to_yaml(self, stream=None): + @overload + def to_yaml(self, stream: IO[str]) -> None: ... + + def to_yaml(self, stream: Optional[IO[str]] = None) -> Optional[str]: return syaml.dump(self.to_dict(), stream) - def to_dict(self): + def to_dict(self) -> Union[str, dict]: + # Mirrors configured as a plain URL are stored as a string in the config schema return self._data - def display(self, max_len=0): + def display(self, max_len: int = 0) -> None: fetch, push = self.fetch_url, self.push_url # don't print the same URL twice url = fetch if fetch == push else f"fetch: {fetch} push: {push}" @@ -102,15 +98,15 @@ def display(self, max_len=0): print(f"{self.name: <{max_len}} [{source}{binary}] {url}") @property - def name(self): + def name(self) -> str: return self._name or "" @property - def binary(self): + def binary(self) -> bool: return isinstance(self._data, str) or self._data.get("binary", True) @property - def source(self): + def source(self) -> bool: return isinstance(self._data, str) or self._data.get("source", True) @property @@ -129,26 +125,26 @@ def autopush(self) -> bool: return self._data.get("autopush", False) @property - def fetch_url(self): + def fetch_url(self) -> str: """Get the valid, canonicalized fetch URL""" return self.get_url("fetch") @property - def push_url(self): - """Get the valid, canonicalized fetch URL""" + def push_url(self) -> str: + """Get the valid, canonicalized push URL""" return self.get_url("push") @property - def fetch_view(self): - """Get the valid, canonicalized fetch URL""" - return self.get_view("fetch") + def fetch_view(self) -> Optional[str]: + """Get the fetch view""" + return self._get_value("view", direction="fetch") @property - def push_view(self): - """Get the valid, canonicalized fetch URL""" - return self.get_view("push") + def push_view(self) -> Optional[str]: + """Get the push view""" + return self._get_value("view", direction="push") - def ensure_mirror_usable(self, direction: str = "push"): + def ensure_mirror_usable(self, direction: str = "push") -> None: access_pair = self._get_value("access_pair", direction) access_token_variable = self._get_value("access_token_variable", direction) @@ -196,7 +192,7 @@ def supported_layout_versions(self) -> List[int]: return supported_versions - def _update_connection_dict(self, current_data: dict, new_data: dict, top_level: bool): + def _update_connection_dict(self, current_data: dict, new_data: dict, top_level: bool) -> bool: # Only allow one to exist in the config if "access_token" in current_data and "access_token_variable" in new_data: current_data.pop("access_token") @@ -236,7 +232,7 @@ def _update_connection_dict(self, current_data: dict, new_data: dict, top_level: changed = True return changed - def update(self, data: dict, direction: Optional[str] = None) -> bool: + def update(self, data: Dict[str, Any], direction: Optional[str] = None) -> bool: """Modify the mirror with the given data. This takes care of expanding trivial mirror definitions by URL to something more rich with a dict if necessary @@ -300,7 +296,7 @@ def update(self, data: dict, direction: Optional[str] = None) -> bool: return self._update_connection_dict(self._data[direction], data, top_level=False) - def _get_value(self, attribute: str, direction: str): + def _get_value(self, attribute: str, direction: str) -> Any: """Returns the most specific value for a given attribute (either push/fetch or global)""" if direction not in ("fetch", "push"): raise ValueError(f"direction must be either 'fetch' or 'push', not {direction}") @@ -342,9 +338,6 @@ def get_url(self, direction: str) -> str: return _url_or_path_to_url(url) - def get_view(self, direction: str): - return self._get_value("view", direction) - def get_credentials(self, direction: str) -> Dict[str, Any]: """Get the mirror credentials from the mirror config @@ -379,9 +372,7 @@ def get_access_token(self, direction: str) -> Optional[str]: tok = self._get_value("access_token_variable", direction) if tok: return os.environ.get(tok) - else: - return self._get_value("access_token", direction) - return None + return self._get_value("access_token", direction) def get_access_pair(self, direction: str) -> Optional[Tuple[str, str]]: pair = self._get_value("access_pair", direction) @@ -406,8 +397,8 @@ class MirrorCollection(Mapping[str, Mirror]): def __init__( self, - mirrors=None, - scope=None, + mirrors: Optional[Mapping[str, Any]] = None, + scope: Optional[str] = None, binary: Optional[bool] = None, source: Optional[bool] = None, autopush: Optional[bool] = None, @@ -444,30 +435,12 @@ def _filter(m: Mirror): self._mirrors = {m.name: m for m in mirrors if _filter(m)} - def __eq__(self, other): + def __eq__(self, other: object) -> bool: + if not isinstance(other, MirrorCollection): + return NotImplemented return self._mirrors == other._mirrors - def to_json(self, stream=None): - return sjson.dump(self.to_dict(True), stream) - - def to_yaml(self, stream=None): - return syaml.dump(self.to_dict(True), stream) - - # TODO: this isn't called anywhere - @staticmethod - def from_yaml(stream, name=None): - data = syaml.load(stream) - return MirrorCollection(data) - - @staticmethod - def from_json(stream, name=None): - try: - d = sjson.load(stream) - return MirrorCollection(d) - except Exception as e: - raise sjson.SpackJSONError("error parsing JSON mirror collection:", str(e)) from e - - def to_dict(self, recursive=False): + def to_dict(self, recursive: bool = False) -> Dict[str, Any]: return syaml.syaml_dict( sorted( ((k, (v.to_dict() if recursive else v)) for (k, v) in self._mirrors.items()), @@ -476,18 +449,20 @@ def to_dict(self, recursive=False): ) @staticmethod - def from_dict(d): + def from_dict(d: Mapping[str, Any]) -> "MirrorCollection": return MirrorCollection(d) - def __getitem__(self, item): + def __getitem__(self, item: str) -> Mirror: return self._mirrors[item] - def display(self): + def display(self) -> None: + if not self._mirrors: + return max_len = max(len(mirror.name) for mirror in self._mirrors.values()) for mirror in self._mirrors.values(): mirror.display(max_len) - def lookup(self, name_or_url): + def lookup(self, name_or_url: str) -> Mirror: """Looks up and returns a Mirror. If this MirrorCollection contains a named Mirror under the name @@ -498,12 +473,12 @@ def lookup(self, name_or_url): result = self.get(name_or_url) if result is None: - result = Mirror(fetch=name_or_url) + result = Mirror(name_or_url) return result - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self._mirrors) - def __len__(self): + def __len__(self) -> int: return len(self._mirrors) diff --git a/lib/spack/spack/mirrors/utils.py b/lib/spack/spack/mirrors/utils.py index ac720a1b96d11f..acb55788af26c2 100644 --- a/lib/spack/spack/mirrors/utils.py +++ b/lib/spack/spack/mirrors/utils.py @@ -15,6 +15,7 @@ from spack.error import MirrorError from spack.llnl.util.filesystem import mkdirp from spack.mirrors.mirror import Mirror, MirrorCollection +from spack.package import InstallError def get_all_versions(specs): @@ -209,6 +210,11 @@ def create_mirror_from_package_object( True if the spec was added successfully, False otherwise """ tty.msg("Adding package {} to mirror".format(pkg_obj.spec.format("{name}{@version}"))) + # Skip placeholder packages + try: + pkg_obj.fetcher + except InstallError: + return False max_retries = 3 for num_retries in range(max_retries): try: diff --git a/lib/spack/spack/mixins.py b/lib/spack/spack/mixins.py index 257be86a4e1781..db5dc0fd0831d6 100644 --- a/lib/spack/spack/mixins.py +++ b/lib/spack/spack/mixins.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This module contains additional behavior that can be attached to any given package.""" + import os from typing import Optional diff --git a/lib/spack/spack/modules/common.py b/lib/spack/spack/modules/common.py index 7c7a1a5b2df686..0d90432808f5d7 100644 --- a/lib/spack/spack/modules/common.py +++ b/lib/spack/spack/modules/common.py @@ -23,6 +23,7 @@ Each of the four classes needs to be sub-classed when implementing a new module type. """ + import collections import contextlib import copy diff --git a/lib/spack/spack/modules/lmod.py b/lib/spack/spack/modules/lmod.py index 1f958b0d15d0fe..8d6c5f32d7bb01 100644 --- a/lib/spack/spack/modules/lmod.py +++ b/lib/spack/spack/modules/lmod.py @@ -14,7 +14,6 @@ import spack.error import spack.llnl.util.filesystem as fs import spack.llnl.util.lang as lang -import spack.repo import spack.spec import spack.tengine as tengine import spack.util.environment @@ -104,20 +103,23 @@ def __init__(self, spec: spack.spec.Spec, module_set_name: str, explicit: bool) super().__init__(spec, module_set_name, explicit) candidates = collections.defaultdict(list) - for node in spec.traverse(deptype=("link", "run")): - candidates["c"].extend(node.dependencies(virtuals=("c",))) - candidates["cxx"].extend(node.dependencies(virtuals=("c",))) - - if candidates["c"]: - self.compiler = candidates["c"][0] - if len(set(candidates["c"])) > 1: - warnings.warn( - f"{spec.short_spec} uses more than one compiler, and might not fit the " - f"LMod hierarchy. Using {self.compiler.short_spec} as the LMod compiler." - ) + language_virtuals = ("c", "cxx", "fortran") - elif not candidates["c"]: - self.compiler = None + for node in spec.traverse(deptype=("link", "run")): + for language in language_virtuals: + candidates[language].extend(node.dependencies(virtuals=(language,))) + + self.compiler = None + + for language in language_virtuals: + if candidates[language]: + self.compiler = candidates[language][0] + if len(set(candidates[language])) > 1: + warnings.warn( + f"{spec.short_spec} uses more than one compiler, and might not fit the " + f"LMod hierarchy. Using {self.compiler.short_spec} as the LMod compiler." + ) + break @property def core_compilers(self) -> List[spack.spec.Spec]: @@ -158,15 +160,6 @@ def hierarchy_tokens(self): """ tokens = configuration(self.name).get("hierarchy", []) - # Check if all the tokens in the hierarchy are virtual specs. - # If not warn the user and raise an error. - not_virtual = [t for t in tokens if t != "compiler" and not spack.repo.PATH.is_virtual(t)] - if not_virtual: - msg = "Non-virtual specs in 'hierarchy' list for lmod: {0}\n" - msg += "Please check the 'modules.yaml' configuration files" - msg = msg.format(", ".join(not_virtual)) - raise NonVirtualInHierarchyError(msg) - # Append 'compiler' which is always implied tokens.append("compiler") @@ -183,10 +176,7 @@ def requires(self): The ``compiler`` key is always present among the requirements. """ # If it's a core_spec, lie and say it requires a core compiler - if ( - any(self.spec.satisfies(core_spec) for core_spec in self.core_specs) - or self.compiler is None - ): + if any(self.spec.satisfies(core_spec) for core_spec in self.core_specs): return {"compiler": self.core_compilers[0]} hierarchy_filter_list = [] @@ -197,17 +187,18 @@ def requires(self): # Keep track of the requirements that this package has in terms # of virtual packages that participate in the hierarchical structure + requirements = {"compiler": self.compiler or self.core_compilers[0]} - requirements = {"compiler": self.compiler} - # For each virtual dependency in the hierarchy + # For each dependency in the hierarchy for x in self.hierarchy_tokens: # Skip anything filtered for this spec if x in hierarchy_filter_list: continue # If I depend on it - if x in self.spec and not self.spec.package.provides(x): + if x in self.spec and not (self.spec.name == x or self.spec.package.provides(x)): requirements[x] = self.spec[x] # record the actual provider + return requirements @property @@ -230,7 +221,7 @@ def provides(self): # All the other tokens in the hierarchy must be virtual dependencies for x in self.hierarchy_tokens: - if self.spec.package.provides(x): + if self.spec.name == x or self.spec.package.provides(x): provides[x] = self.spec return provides @@ -255,7 +246,9 @@ def missing(self): @property def hidden(self): # Never hide a module that opens a hierarchy - if any(self.spec.package.provides(x) for x in self.hierarchy_tokens): + if any( + self.spec.name == x or self.spec.package.provides(x) for x in self.hierarchy_tokens + ): return False return super().hidden @@ -509,9 +502,3 @@ class CoreCompilersNotFoundError(spack.error.SpackError, KeyError): """Error raised if the key ``core_compilers`` has not been specified in the configuration file. """ - - -class NonVirtualInHierarchyError(spack.error.SpackError, TypeError): - """Error raised if non-virtual specs are used as hierarchy tokens in - the lmod section of ``modules.yaml``. - """ diff --git a/lib/spack/spack/modules/tcl.py b/lib/spack/spack/modules/tcl.py index 78542b2568ccbe..a1468331578a87 100644 --- a/lib/spack/spack/modules/tcl.py +++ b/lib/spack/spack/modules/tcl.py @@ -5,6 +5,7 @@ """This module implements the classes necessary to generate Tcl non-hierarchical modules. """ + import os from typing import Dict, Optional, Tuple diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py index df96f4870b62ae..3fda8115283138 100644 --- a/lib/spack/spack/multimethod.py +++ b/lib/spack/spack/multimethod.py @@ -23,9 +23,10 @@ depending on the scenario, regular old conditionals might be clearer, so package authors should use their judgement. """ + import functools from contextlib import contextmanager -from typing import Union +from typing import Optional, Union import spack.directives_meta import spack.error @@ -237,6 +238,8 @@ def install(self, prefix): override all of the decorated versions. This is a limitation of the Python language. """ + spec: Optional[spack.spec.Spec] + def __init__(self, condition: Union[str, bool]): """Can be used both as a decorator, for multimethods, or as a context manager to group ``when=`` arguments together. @@ -245,31 +248,33 @@ def __init__(self, condition: Union[str, bool]): Args: condition (str): condition to be met """ - if isinstance(condition, bool): - self.spec = spack.spec.Spec() if condition else None - else: - self.spec = spack.spec.Spec(condition) + self.when = condition def __call__(self, method): - assert ( - MultiMethodMeta._locals is not None - ), "cannot use multimethod, missing MultiMethodMeta metaclass?" + assert MultiMethodMeta._locals is not None, ( + "cannot use multimethod, missing MultiMethodMeta metaclass?" + ) # Create a multimethod with this name if there is not one already original_method = MultiMethodMeta._locals.get(method.__name__) if not isinstance(original_method, SpecMultiMethod): original_method = SpecMultiMethod(original_method) - if self.spec is not None: - original_method.register(self.spec, method) + if self.when is True: + original_method.register(spack.spec.EMPTY_SPEC, method) + elif self.when is not False: + original_method.register(spack.directives_meta.get_spec(self.when), method) return original_method def __enter__(self): - spack.directives_meta.DirectiveMeta.push_to_context(str(self.spec)) + # TODO: support when=False. + if isinstance(self.when, str): + spack.directives_meta.DirectiveMeta.push_when_constraint(self.when) def __exit__(self, exc_type, exc_val, exc_tb): - spack.directives_meta.DirectiveMeta.pop_from_context() + if isinstance(self.when, str): + spack.directives_meta.DirectiveMeta.pop_when_constraint() @contextmanager diff --git a/lib/spack/spack/new_installer.py b/lib/spack/spack/new_installer.py index 404da25f7dfc2b..5b7c92251a26f6 100644 --- a/lib/spack/spack/new_installer.py +++ b/lib/spack/spack/new_installer.py @@ -20,13 +20,18 @@ runs an event loop to listen for control messages from the UI process (to enable/disable echoing of logs), and for output from the build process.""" +import codecs import fcntl +import glob import io import json +import multiprocessing import os import re import selectors +import shlex import shutil +import signal import sys import tempfile import termios @@ -34,10 +39,23 @@ import time import traceback import tty +import warnings from gzip import GzipFile from multiprocessing import Pipe, Process from multiprocessing.connection import Connection -from typing import TYPE_CHECKING, Callable, Dict, Generator, List, Optional, Set, Tuple, Union +from typing import ( + TYPE_CHECKING, + Callable, + Dict, + FrozenSet, + Generator, + List, + NamedTuple, + Optional, + Set, + Tuple, + Union, +) from spack.vendor.typing_extensions import Literal @@ -49,16 +67,27 @@ import spack.deptypes as dt import spack.error import spack.hooks -import spack.llnl.util.lock +import spack.llnl.util.filesystem as fs import spack.llnl.util.tty +import spack.llnl.util.tty.color +import spack.mirrors.mirror import spack.paths import spack.report +import spack.sandbox import spack.spec import spack.stage import spack.store +import spack.subprocess_context import spack.traverse import spack.url_buildcache +import spack.util.environment import spack.util.lock +from spack.installer import _do_fake_install, dump_packages +from spack.llnl.util.lang import pretty_duration +from spack.llnl.util.tty.log import _is_background_tty, ignore_signal +from spack.util.executable import ProcessError +from spack.util.log_parse import make_log_context, parse_log_events +from spack.util.path import padding_filter, padding_filter_bytes if TYPE_CHECKING: import spack.package_base @@ -69,11 +98,17 @@ #: How often to update a spinner in seconds SPINNER_INTERVAL = 0.1 +#: How often to wake up in headless mode to check for background->foreground transition (seconds) +HEADLESS_WAKE_INTERVAL = 1.0 + #: How long to display finished packages before graying them out CLEANUP_TIMEOUT = 2.0 +#: How often to flush completed builds to the database +DATABASE_WRITE_INTERVAL = 5.0 + #: Size of the output buffer for child processes -OUTPUT_BUFFER_SIZE = 4096 +OUTPUT_BUFFER_SIZE = 32768 #: Suffix for temporary backup during overwrite install OVERWRITE_BACKUP_SUFFIX = ".old" @@ -82,10 +117,52 @@ OVERWRITE_GARBAGE_SUFFIX = ".garbage" -class ChildInfo: +class ExitCode: + SUCCESS = 0 + BUILD_ERROR = 1 + #: Exit code used by the child process to signal that the build was stopped at a phase boundary + STOPPED_AT_PHASE = 3 + #: Exit code used by the child process to signal a binary cache miss (no source fallback) + BUILD_CACHE_MISS = 4 + + +class DatabaseAction: + """Base class for objects that need to be persisted to the database.""" + + __slots__ = ("spec", "prefix_lock") + + spec: "spack.spec.Spec" + prefix_lock: Optional[spack.util.lock.Lock] + + def save_to_db(self, db: spack.database.Database) -> None: ... + + def release_prefix_lock(self) -> None: + if self.prefix_lock is not None: + try: + self.prefix_lock.release_write() + except Exception: + pass + self.prefix_lock = None + + +class MarkExplicitAction(DatabaseAction): + """Action to mark an already installed spec as explicitly installed. Similar to ChildInfo, but + used when no build process was needed.""" + + __slots__ = () + + def __init__(self, spec: "spack.spec.Spec") -> None: + self.spec = spec + self.prefix_lock = None + + def save_to_db(self, db: spack.database.Database) -> None: + db._mark(self.spec, "explicit", True) + + +class ChildInfo(DatabaseAction): """Information about a child process.""" - __slots__ = ("proc", "spec", "output_r_conn", "state_r_conn", "control_w_conn", "explicit") + __slots__ = ("proc", "output_r_conn", "state_r_conn", "control_w_conn", "explicit", "log_path") def __init__( self, @@ -94,6 +171,7 @@ def __init__( output_r_conn: Connection, state_r_conn: Connection, control_w_conn: Connection, + log_path: str, explicit: bool = False, ) -> None: self.proc = proc @@ -101,10 +179,16 @@ def __init__( self.output_r_conn = output_r_conn self.state_r_conn = state_r_conn self.control_w_conn = control_w_conn + self.log_path = log_path self.explicit = explicit + self.prefix_lock: Optional[spack.util.lock.Lock] = None - def cleanup(self, selector: selectors.BaseSelector) -> None: - """Unregister and close file descriptors, and join the child process.""" + def save_to_db(self, db: spack.database.Database) -> None: + return db._add(self.spec, explicit=self.explicit) + + def close(self, selector: selectors.BaseSelector) -> int: + """Unregister and close file descriptors, and join the child process. + Returns the exit code of the child process.""" try: selector.unregister(self.output_r_conn.fileno()) except KeyError: @@ -121,6 +205,11 @@ def cleanup(self, selector: selectors.BaseSelector) -> None: self.state_r_conn.close() self.control_w_conn.close() self.proc.join() + exit_code = self.proc.exitcode + assert exit_code is not None, "Finished build should have exit code set" + if hasattr(self.proc, "close"): # No known equivalent in Python 3.6 + self.proc.close() + return exit_code def send_state(state: str, state_pipe: io.TextIOWrapper) -> None: @@ -135,7 +224,13 @@ def send_progress(current: int, total: int, state_pipe: io.TextIOWrapper) -> Non state_pipe.write("\n") -def tee(control_r: int, log_r: int, file_w: int, parent_w: int) -> None: +def send_installed_from_binary_cache(state_pipe: io.TextIOWrapper) -> None: + """Send a notification that the package was installed from binary cache.""" + json.dump({"installed_from_binary_cache": True}, state_pipe, separators=(",", ":")) + state_pipe.write("\n") + + +def tee(control_r: int, log_r: int, log_file: io.BufferedWriter, parent_w: int) -> None: """Forward log_r to file_w and parent_w (if echoing is enabled). Echoing is enabled and disabled by reading from control_r.""" echo_on = False @@ -144,22 +239,25 @@ def tee(control_r: int, log_r: int, file_w: int, parent_w: int) -> None: selector.register(control_r, selectors.EVENT_READ) try: - while True: - for key, _ in selector.select(): - if key.fd == log_r: - data = os.read(log_r, OUTPUT_BUFFER_SIZE) - if not data: # EOF: exit the thread - return - os.write(file_w, data) - if echo_on: - os.write(parent_w, data) - - elif key.fd == control_r: - control_data = os.read(control_r, 1) - if not control_data: - return - else: - echo_on = control_data == b"1" + with log_file, open(parent_w, "wb", closefd=False) as parent: + while True: + for key, _ in selector.select(): + if key.fd == log_r: + data = os.read(log_r, OUTPUT_BUFFER_SIZE) + if not data: # EOF: exit the thread + return + log_file.write(data) + log_file.flush() + if echo_on: + parent.write(data) + parent.flush() + + elif key.fd == control_r: + control_data = os.read(control_r, 1) + if not control_data: + return + else: + echo_on = control_data == b"1" except OSError: # do not raise pass finally: @@ -170,34 +268,41 @@ class Tee: """Emulates ./build 2>&1 | tee build.log. The output is sent both to a log file and the parent process (if echoing is enabled). The control_fd is used to enable/disable echoing.""" - def __init__(self, control: Connection, parent: Connection, log_fd: int) -> None: + def __init__(self, control: Connection, parent: Connection, log_path: str) -> None: self.control = control self.parent = parent - #: The file descriptor of the log file - self.log_fd = log_fd + # sys.stdout and sys.stderr may have been replaced with file objects under pytest, so + # redirect their file descriptors in addition to the original fds 1 and 2. + fds = {sys.stdout.fileno(), sys.stderr.fileno(), 1, 2} + self.saved_fds = {fd: os.dup(fd) for fd in fds} + #: The path of the log file + self.log_path = log_path + log_file = open(self.log_path, "ab") r, w = os.pipe() self.tee_thread = threading.Thread( target=tee, - args=(self.control.fileno(), r, self.log_fd, self.parent.fileno()), + args=(self.control.fileno(), r, log_file, self.parent.fileno()), daemon=True, ) self.tee_thread.start() - os.dup2(w, sys.stdout.fileno()) - os.dup2(w, sys.stderr.fileno()) + for fd in fds: + os.dup2(w, fd) os.close(w) def close(self) -> None: # Closing stdout and stderr should close the last reference to the write end of the pipe, - # causing the tee thread to wake up, flush the last data, and exit. + # causing the tee thread to wake up, flush the last data, and exit. We restore stdout and + # stderr, because between sys.exit and the actual process exit buffers may be flushed, and + # can cause exit code 120 (witnessed under pytest+coverage on macOS). sys.stdout.flush() sys.stderr.flush() - os.close(sys.stdout.fileno()) - os.close(sys.stderr.fileno()) + for fd, saved_fd in self.saved_fds.items(): + os.dup2(saved_fd, fd) + os.close(saved_fd) self.tee_thread.join() # Only then close the other fds. self.control.close() self.parent.close() - os.close(self.log_fd) def install_from_buildcache( @@ -207,7 +312,12 @@ def install_from_buildcache( state_stream: io.TextIOWrapper, ) -> bool: send_state("fetching from build cache", state_stream) - tarball_stage = spack.binary_distribution.download_tarball(spec.build_spec, unsigned, mirrors) + try: + tarball_stage = spack.binary_distribution.download_tarball( + spec.build_spec, unsigned, mirrors + ) + except spack.binary_distribution.NoConfiguredBinaryMirrors: + return False if tarball_stage is None: return False @@ -224,26 +334,58 @@ def install_from_buildcache( pkg._post_buildcache_install_hook() pkg.installed_from_binary_cache = True + # inform also the parent that this package was installed from binary cache. + send_installed_from_binary_cache(state_stream) + return True +class GlobalState: + """Global state needed in a build subprocess. This is similar to spack.subprocess_context, + but excludes the Spack environment, which is slow to serialize and should not be needed + during the build.""" + + __slots__ = ("store", "config", "monkey_patches", "spack_working_dir", "repo_cache") + + def __init__(self): + if multiprocessing.get_start_method() == "fork": + return + self.config = spack.config.CONFIG.ensure_unwrapped() + self.store = spack.store.STORE + self.monkey_patches = spack.subprocess_context.TestPatches.create() + self.spack_working_dir = spack.paths.spack_working_dir + + def restore(self): + if multiprocessing.get_start_method() == "fork": + # In the forking case we must erase SSL contexts. + from spack.oci import opener + from spack.util import web + from spack.util.s3 import s3_client_cache + + web.urlopen._instance = None + opener.urlopen._instance = None + s3_client_cache.clear() + return + spack.store.STORE = self.store + spack.config.CONFIG = self.config + self.monkey_patches.restore() + spack.paths.spack_working_dir = self.spack_working_dir + + class PrefixPivoter: - """Manages the installation prefix during overwrite installations.""" + """Manages the installation prefix of a build.""" - def __init__(self, prefix: str, overwrite: bool, keep_prefix: bool = False) -> None: + def __init__(self, prefix: str, keep_prefix: bool = False) -> None: """Initialize the prefix pivoter. Args: prefix: The installation prefix path - overwrite: Whether to allow overwriting an existing prefix - keep_prefix: Whether to keep a failed installation prefix (when not overwriting) + keep_prefix: Whether to keep a failed installation prefix """ self.prefix = prefix - #: Whether to allow installation when the prefix exists - self.overwrite = overwrite #: Whether to keep a failed installation prefix self.keep_prefix = keep_prefix - #: Temporary location for the original prefix during overwrite + #: Temporary location for the original prefix self.tmp_prefix: Optional[str] = None self.parent = os.path.dirname(prefix) @@ -251,9 +393,7 @@ def __enter__(self) -> "PrefixPivoter": """Enter the context: move existing prefix to temporary location if needed.""" if not self._lexists(self.prefix): return self - if not self.overwrite: - raise spack.error.InstallError(f"Install prefix {self.prefix} already exists") - # Move the existing prefix to a temporary location + # Move the existing prefix to a temporary location so the build starts fresh self.tmp_prefix = self._mkdtemp( dir=self.parent, prefix=".", suffix=OVERWRITE_BACKUP_SUFFIX ) @@ -265,36 +405,33 @@ def __exit__( ) -> None: """Exit the context: cleanup on success, restore on failure.""" if exc_type is None: - # Success: remove the backup in case of overwrite + # Success: remove the backup if self.tmp_prefix is not None: self._rmtree_ignore_errors(self.tmp_prefix) return # Failure handling: - # Priority 1: If we're overwriting, always restore the original prefix - # Priority 2: If keep_prefix is False, remove the failed installation - - if self.overwrite and self.tmp_prefix is not None: - # Overwrite case: restore the original prefix if it existed - # The highest priority is to restore the original prefix, so we try to: - # rename prefix -> garbage: move failed dir out of the way - # rename tmp_prefix -> prefix: restore original prefix - # remove garbage (this is allowed to fail) + if self.keep_prefix and not issubclass(exc_type, BinaryCacheMiss): + # Leave the failed prefix in place, discard the backup. Except for binary cache misses, + # which is a scheduling failure and not a build failure. + if self.tmp_prefix is not None: + self._rmtree_ignore_errors(self.tmp_prefix) + elif self.tmp_prefix is not None: + # There was a pre-existing prefix: pivot back to it and discard the failed build garbage = self._mkdtemp(dir=self.parent, prefix=".", suffix=OVERWRITE_GARBAGE_SUFFIX) try: self._rename(self.prefix, garbage) has_failed_prefix = True - except FileNotFoundError: # prefix dir does not exist, so we don't have to delete it. + except FileNotFoundError: # build never created the prefix dir has_failed_prefix = False self._rename(self.tmp_prefix, self.prefix) if has_failed_prefix: self._rmtree_ignore_errors(garbage) - elif not self.keep_prefix and self._lexists(self.prefix): - # Not overwriting, keep_prefix is False: remove the failed installation + elif self._lexists(self.prefix): + # No backup, just remove the failed installation garbage = self._mkdtemp(dir=self.parent, prefix=".", suffix=OVERWRITE_GARBAGE_SUFFIX) self._rename(self.prefix, garbage) self._rmtree_ignore_errors(garbage) - # else: keep_prefix is True, leave the failed prefix in place def _lexists(self, path: str) -> bool: return os.path.lexists(path) @@ -318,17 +455,21 @@ def worker_function( dirty: bool, keep_stage: bool, restage: bool, - overwrite: bool, keep_prefix: bool, skip_patch: bool, + fake: bool, + install_source: bool, + run_tests: bool, state: Connection, parent: Connection, echo_control: Connection, makeflags: str, js1: Optional[Connection], js2: Optional[Connection], - store: spack.store.Store, - config: spack.config.Configuration, + log_path: str, + global_state: GlobalState, + stop_before: Optional[str] = None, + stop_at: Optional[str] = None, ): """ Function run in the build child process. Installs the specified spec, sending state updates @@ -343,42 +484,79 @@ def worker_function( dirty: Whether to preserve user environment in the build environment keep_stage: Whether to keep the build stage after installation restage: Whether to restage the source before building - overwrite: Whether to overwrite the existing install prefix keep_prefix: Whether to keep a failed installation prefix skip_patch: Whether to skip the patch phase + run_tests: Whether to run install-time tests for this package state: Connection to send state updates to parent: Connection to send build output to echo_control: Connection to receive echo control messages from makeflags: MAKEFLAGS to set, so that the build process uses the POSIX jobserver js1: Connection for old style jobserver read fd (if any). Unused, just to inherit fd. js2: Connection for old style jobserver write fd (if any). Unused, just to inherit fd. - store: global store instance from parent - config: global config instance from parent + log_path: Path to the log file to write build output to + global_state: Global state to restore """ # TODO: don't start a build for external packages if spec.external: return + global_state.restore() + + # Isolate the process group to shield against Ctrl+C and enable safe killpg() cleanup. In + # constrast to setsid(), this keeps a neat process group hierarchy for utils like pstree. + os.setpgid(0, 0) + + # Reset SIGTSTP to default in case the parent had a custom handler. + signal.signal(signal.SIGTSTP, signal.SIG_DFL) + + def handle_sigterm(signum, frame): + # This SIGTERM handler forwards the signal to child processes (cmake, make, etc). We wait + # for all child processes to exit before raising KeyboardInterrupt. This ensures all + # __exit__ and finally blocks run after the child processes have stopped, meaning that we + # get to clean up the prefix without risking that the child process writes to it + # afterwards. + signal.signal(signal.SIGTERM, signal.SIG_IGN) + os.killpg(0, signal.SIGTERM) + + try: + while True: + os.waitpid(-1, 0) + except ChildProcessError: + pass + + raise KeyboardInterrupt("Installation interrupted") + + signal.signal(signal.SIGTERM, handle_sigterm) + os.environ["MAKEFLAGS"] = makeflags - spack.store.STORE = store - spack.config.CONFIG = config - spack.paths.set_working_dir() - - # Create a log file in the root of the stage dir. - log_fd, log_path = tempfile.mkstemp( - prefix=f"spack-stage-{spec.name}-{spec.dag_hash()}-", - suffix=".log", - dir=spack.stage.get_stage_root(), + + # Force line buffering for Python's textio wrappers of stdout/stderr. We're not going to print + # much ourselves, but what we print should appear before output from `make` and other build + # tools. + sys.stdout = os.fdopen( + sys.stdout.fileno(), "w", buffering=1, encoding=sys.stdout.encoding, closefd=False ) - tee = Tee(echo_control, parent, log_fd) + sys.stderr = os.fdopen( + sys.stderr.fileno(), "w", buffering=1, encoding=sys.stderr.encoding, closefd=False + ) + + # Detach stdin from the terminal like `./build < /dev/null`. This would not be necessary if we + # used os.setsid() instead of os.setpgid(), but that would "break" pstree output. + devnull_fd = os.open(os.devnull, os.O_RDONLY) + os.dup2(devnull_fd, 0) + os.close(devnull_fd) + sys.stdin = open(os.devnull, "r", encoding=sys.stdin.encoding) + + # Start the tee thread to forward output to the log file and parent process. + tee = Tee(echo_control, parent, log_path) # Use closedfd=false because of the connection objects. Use line buffering. state_stream = os.fdopen(state.fileno(), "w", buffering=1, closefd=False) - exit_code = 0 + exit_code = ExitCode.SUCCESS try: - with PrefixPivoter(spec.prefix, overwrite, keep_prefix): + with PrefixPivoter(spec.prefix, keep_prefix): _install( spec, explicit, @@ -389,32 +567,153 @@ def worker_function( keep_stage, restage, skip_patch, + fake, + install_source, state_stream, log_path, - store, + spack.store.STORE, + run_tests, + stop_before, + stop_at, ) - except Exception: - traceback.print_exc() # log the traceback to the log file - exit_code = 1 + except spack.error.StopPhase: + exit_code = ExitCode.STOPPED_AT_PHASE + except ProcessError as e: + print(e, file=sys.stderr) + exit_code = ExitCode.BUILD_ERROR + except BinaryCacheMiss: + exit_code = ExitCode.BUILD_CACHE_MISS + except BaseException: + traceback.print_exc(limit=-4) + exit_code = ExitCode.BUILD_ERROR finally: tee.close() state_stream.close() - if exit_code == 0 and not os.path.lexists(spec.package.install_log_path): + if exit_code == ExitCode.SUCCESS: # Try to install the compressed log file - try: - with open(log_path, "rb") as f, open(spec.package.install_log_path, "wb") as g: - # Use GzipFile directly so we can omit filename / mtime in header - gzip_file = GzipFile(filename="", mode="wb", compresslevel=6, mtime=0, fileobj=g) - shutil.copyfileobj(f, gzip_file) - gzip_file.close() - os.unlink(log_path) - except Exception: - pass # don't fail the build just because log compression failed + if not os.path.lexists(spec.package.install_log_path): + try: + with open(log_path, "rb") as f, open(spec.package.install_log_path, "wb") as g: + # Use GzipFile directly so we can omit filename / mtime in header + gzip_file = GzipFile( + filename="", mode="wb", compresslevel=6, mtime=0, fileobj=g + ) + shutil.copyfileobj(f, gzip_file) + gzip_file.close() + except Exception: + pass # don't fail the build just because log compression failed sys.exit(exit_code) +def _archive_build_metadata(pkg: "spack.package_base.PackageBase") -> None: + """Copy build metadata from stage to install prefix .spack directory. + + Mirrors what the old installer's log() function does in the parent process. + Only called after a successful source build (not for binary cache installs). + Errors are suppressed to avoid failing the build over metadata archiving.""" + + try: + if os.path.lexists(pkg.env_mods_path): + shutil.copy2(pkg.env_mods_path, pkg.install_env_path) + except OSError as e: + spack.llnl.util.tty.debug(e) + try: + if os.path.lexists(pkg.configure_args_path): + shutil.copy2(pkg.configure_args_path, pkg.install_configure_args_path) + except OSError as e: + spack.llnl.util.tty.debug(e) + + # Archive install-phase test log if present + try: + pkg.archive_install_test_log() + except Exception as e: + spack.llnl.util.tty.debug(e) + + # Archive package-specific files matched by archive_files glob patterns + try: + with fs.working_dir(pkg.stage.path): + target_dir = os.path.join( + spack.store.STORE.layout.metadata_path(pkg.spec), "archived-files" + ) + errors = io.StringIO() + for glob_expr in spack.builder.create(pkg).archive_files: + abs_expr = os.path.realpath(glob_expr) + if os.path.realpath(pkg.stage.path) not in abs_expr: + errors.write(f"[OUTSIDE SOURCE PATH]: {glob_expr}\n") + continue + if os.path.isabs(glob_expr): + glob_expr = os.path.relpath(glob_expr, pkg.stage.path) + for f in glob.glob(glob_expr): + try: + target = os.path.join(target_dir, f) + fs.mkdirp(os.path.dirname(target)) + fs.install(f, target) + except Exception as e: + spack.llnl.util.tty.debug(e) + errors.write(f"[FAILED TO ARCHIVE]: {f}") + if errors.getvalue(): + error_file = os.path.join(target_dir, "errors.txt") + fs.mkdirp(target_dir) + with open(error_file, "w", encoding="utf-8") as err: + err.write(errors.getvalue()) + spack.llnl.util.tty.warn( + f"Errors occurred when archiving files.\n\tSee: {error_file}" + ) + except Exception as e: + spack.llnl.util.tty.debug(e) + + try: + packages_dir = spack.store.STORE.layout.build_packages_path(pkg.spec) + dump_packages(pkg.spec, packages_dir) + except Exception as e: + spack.llnl.util.tty.debug(e) + + try: + spack.store.STORE.layout.write_host_environment(pkg.spec) + except Exception as e: + spack.llnl.util.tty.debug(e) + + +def _enable_sandbox(config: dict, spec: spack.spec.Spec, stage_path: str) -> None: + if not config.get("enable", False): + return + + try: + sandbox = spack.sandbox.get_sandbox() + except spack.sandbox.SandboxError as e: + raise spack.error.InstallError(f"Cannot enable build sandbox: {e}") from e + + for dep in spec.traverse(root=False): + if not dep.external: + sandbox.allow_read(dep.prefix) + + sandbox.allow_write(stage_path) + sandbox.allow_write(spec.prefix) + + # POSIX prescribes /tmp and /dev/null are present. In the future we can consider setting + # TMPPATH to a sibling of the stage path to isolate concurrent builds better. + sandbox.allow_write(tempfile.gettempdir()) + sandbox.allow_write(os.devnull) + + # Allow read access to sbang, which might be needed to run build scripts. + sandbox.allow_read(os.path.join(spack.store.STORE.unpadded_root, "bin", "sbang")) + for upstream_db in spack.store.STORE.upstreams or []: + sandbox.allow_read(os.path.join(upstream_db.root, "bin", "sbang")) + + # User-configured paths + for p in config.get("allow_read", []): + sandbox.allow_read(p) + for p in config.get("allow_write", []): + sandbox.allow_write(p) + + try: + sandbox.apply(block_network=not config.get("allow_network", True)) + except spack.sandbox.SandboxError as e: + raise spack.error.InstallError(f"Cannot enable build sandbox: {e}") from e + + def _install( spec: spack.spec.Spec, explicit: bool, @@ -425,26 +724,38 @@ def _install( keep_stage: bool, restage: bool, skip_patch: bool, + fake: bool, + install_source: bool, state_stream: io.TextIOWrapper, log_path: str, store: spack.store.Store = spack.store.STORE, + run_tests: bool = False, + stop_before: Optional[str] = None, + stop_at: Optional[str] = None, ) -> None: """Install a spec from build cache or source.""" # Create the stage and log file before starting the tee thread. pkg = spec.package + pkg.run_tests = run_tests + + if fake: + store.layout.create_install_directory(spec) + _do_fake_install(pkg) + spack.hooks.post_install(spec, explicit) + return # Try to install from buildcache, unless user asked for source only if install_policy != "source_only": - if mirrors and install_from_buildcache(mirrors, spec, unsigned, state_stream): + if install_from_buildcache(mirrors, spec, unsigned, state_stream): spack.hooks.post_install(spec, explicit) return elif install_policy == "cache_only": - # Binary required but not available send_state("no binary available", state_stream) - raise spack.error.InstallError(f"No binary available for {spec}") + raise BinaryCacheMiss(f"No binary available for {spec}") - spack.build_environment.setup_package(pkg, dirty=dirty) + unmodified_env = os.environ.copy() + env_mods = spack.build_environment.setup_package(pkg, dirty=dirty) store.layout.create_install_directory(spec) stage = pkg.stage @@ -456,6 +767,28 @@ def _install( stage.destroy() stage.create() + # Write build environment and env-mods to stage + spack.util.environment.dump_environment(pkg.env_path) + with open(pkg.env_mods_path, "w", encoding="utf-8") as f: + f.write(env_mods.shell_modifications(explicit=True, env=unmodified_env)) + + # Try to snapshot configure/cmake args before phases run + for attr in ("configure_args", "cmake_args"): + try: + args = getattr(pkg, attr)() + with open(pkg.configure_args_path, "w", encoding="utf-8") as f: + f.write(" ".join(shlex.quote(a) for a in args)) + break + except Exception: + pass + + # For develop packages or non-develop packages with --keep-stage there may be a + # pre-existing symlink at pkg.log_path which would cause the new symlink to fail. + # Try removing it if it exists. + try: + os.unlink(pkg.log_path) + except OSError: + pass os.symlink(log_path, pkg.log_path) send_state("staging", state_stream) @@ -467,12 +800,38 @@ def _install( os.chdir(stage.source_path) + if install_source and os.path.isdir(stage.source_path): + src_target = os.path.join(spec.prefix, "share", spec.name, "src") + fs.install_tree(stage.source_path, src_target) + spack.hooks.pre_install(spec) - for phase in spack.builder.create(pkg): - send_state(phase.name, state_stream) - phase.execute() + builder = spack.builder.create(pkg) + if stop_before is not None and stop_before not in builder.phases: + raise spack.error.InstallError(f"'{stop_before}' is not a valid phase for {pkg.name}") + if stop_at is not None and stop_at not in builder.phases: + raise spack.error.InstallError(f"'{stop_at}' is not a valid phase for {pkg.name}") + _enable_sandbox(spack.config.get("config:sandbox", {}), spec, stage.path) + + for phase in builder: + if stop_before is not None and phase.name == stop_before: + send_state(f"stopped before {stop_before}", state_stream) + raise spack.error.StopPhase(f"Stopping before '{stop_before}'") + send_state(phase.name, state_stream) + spack.llnl.util.tty.msg(f"{pkg.name}: Executing phase: '{phase.name}'") + # Run the install phase with debug output enabled. + old_debug = spack.llnl.util.tty.debug_level() + spack.llnl.util.tty.set_debug(1) + try: + phase.execute() + finally: + spack.llnl.util.tty.set_debug(old_debug) + if stop_at is not None and phase.name == stop_at: + send_state(f"stopped after {stop_at}", state_stream) + raise spack.error.StopPhase(f"Stopping at '{stop_at}'") + + _archive_build_metadata(pkg) spack.hooks.post_install(spec, explicit) @@ -482,7 +841,14 @@ class JobServer: def __init__(self, num_jobs: int) -> None: #: Keep track of how many tokens Spack itself has acquired, which is used to release them. self.tokens_acquired = 0 + #: The number of jobs to run concurrently. This translates to `num_jobs - 1` tokens in the + #: jobserver. self.num_jobs = num_jobs + #: The target number of jobs to run concurrently, which may differ from num_jobs if the + #: user has requested a decrease in parallelism, but we haven't consumed enough tokens to + #: reflect that yet. This value is used in the UI. The invariant is that self.target_jobs + #: can only be modified if self.created is True. + self.target_jobs = num_jobs self.fifo_path: Optional[str] = None self.created = False self._setup() @@ -525,6 +891,38 @@ def makeflags(self, gmake: Optional[spack.spec.Spec]) -> str: else: return f" -j{self.num_jobs} --jobserver-fds={self.r},{self.w}" + def has_target_parallelism(self) -> bool: + return self.num_jobs == self.target_jobs + + def increase_parallelism(self) -> None: + """Add one token to the jobserver to increase parallelism; this should always work.""" + if not self.created: + return + self.target_jobs += 1 + # If a decrease was pending, don't add a token. + if self.target_jobs <= self.num_jobs: + return + os.write(self.w, b"+") + self.num_jobs += 1 + + def decrease_parallelism(self) -> None: + """Request an eventual concurrency decrease by 1.""" + if not self.created or self.target_jobs <= 1: + return + self.target_jobs -= 1 + self.maybe_discard_tokens() + + def maybe_discard_tokens(self) -> None: + """Try to get reduce parallelism by discarding tokens.""" + to_discard = self.num_jobs - self.target_jobs + if to_discard <= 0: + return + try: + # The read may return zero or just fewer bytes than requested; we'll try again later. + self.num_jobs -= len(os.read(self.r, to_discard)) + except BlockingIOError: + pass + def acquire(self, jobs: int) -> int: """Try and acquire at most 'jobs' tokens from the jobserver. Returns the number of tokens actually acquired (may be less than requested, or zero).""" @@ -540,10 +938,34 @@ def release(self) -> None: # The last job to quit has an implicit token, so don't release if we have none. if self.tokens_acquired == 0: return - os.write(self.w, b"+") self.tokens_acquired -= 1 + if self.target_jobs < self.num_jobs: + # If a decrease in parallelism is requested, discard a token instead of releasing it. + self.num_jobs -= 1 + else: + os.write(self.w, b"+") def close(self) -> None: + if self.created and self.num_jobs > 1: + if self.tokens_acquired != 0: + # It's a non-fatal internal error to close the jobserver with acquired tokens. + warnings.warn("Spack failed to release jobserver tokens", stacklevel=2) + else: + # Verify that all build processes released the tokens they acquired. + total = self.num_jobs - 1 + drained = self.acquire(total) + if drained != total: + n = total - drained + warnings.warn( + f"{n} jobserver {'token was' if n == 1 else 'tokens were'} not released " + "by the build processes. This can indicate that the build ran with " + "limited parallelism.", + stacklevel=2, + ) + + self.r_conn.close() + self.w_conn.close() + # Remove the FIFO if we created it. if self.created and self.fifo_path: try: @@ -554,11 +976,6 @@ def close(self) -> None: os.rmdir(os.path.dirname(self.fifo_path)) except OSError: pass - # TODO: implement a sanity check here: - # 1. did we release all tokens we acquired? - # 2. if we created the jobserver, did the children return all tokens? - self.r_conn.close() - self.w_conn.close() def start_build( @@ -570,10 +987,15 @@ def start_build( dirty: bool, keep_stage: bool, restage: bool, - overwrite: bool, keep_prefix: bool, skip_patch: bool, + fake: bool, + install_source: bool, + run_tests: bool, jobserver: JobServer, + log_path: str, + stop_before: Optional[str] = None, + stop_at: Optional[str] = None, ) -> ChildInfo: """Start a new build.""" # Create pipes for the child's output, state reporting, and control. @@ -598,17 +1020,21 @@ def start_build( dirty, keep_stage, restage, - overwrite, keep_prefix, skip_patch, + fake, + install_source, + run_tests, state_w_conn, output_w_conn, control_r_conn, makeflags, None if fifo else jobserver.r_conn, None if fifo else jobserver.w_conn, - spack.store.STORE, - spack.config.CONFIG, + log_path, + GlobalState(), + stop_before, + stop_at, ), ) proc.start() @@ -622,7 +1048,7 @@ def start_build( os.set_blocking(output_r_conn.fileno(), False) os.set_blocking(state_r_conn.fileno(), False) - return ChildInfo(proc, spec, output_r_conn, state_r_conn, control_w_conn, explicit) + return ChildInfo(proc, spec, output_r_conn, state_r_conn, control_w_conn, log_path, explicit) def get_jobserver_config(makeflags: Optional[str] = None) -> Optional[Union[str, Tuple[int, int]]]: @@ -712,11 +1138,22 @@ class BuildInfo: "external", "prefix", "finished_time", + "start_time", + "duration", "progress_percent", "control_w_conn", + "log_path", + "log_summary", ) - def __init__(self, spec: spack.spec.Spec, explicit: bool, control_w_conn: Connection) -> None: + def __init__( + self, + spec: spack.spec.Spec, + explicit: bool, + control_w_conn: Optional[Connection], + log_path: Optional[str] = None, + start_time: float = 0.0, + ) -> None: self.state: str = "starting" self.explicit: bool = explicit self.version: str = str(spec.version) @@ -725,8 +1162,12 @@ def __init__(self, spec: spack.spec.Spec, explicit: bool, control_w_conn: Connec self.external: bool = spec.external self.prefix: str = spec.prefix self.finished_time: Optional[float] = None + self.start_time: float = start_time + self.duration: Optional[float] = None self.progress_percent: Optional[int] = None self.control_w_conn = control_w_conn + self.log_path = log_path + self.log_summary: Optional[str] = None class BuildStatus: @@ -736,9 +1177,12 @@ def __init__( self, total: int, stdout: io.TextIOWrapper = sys.stdout, # type: ignore[assignment] - get_terminal_size: Callable[[], Tuple[int, int]] = os.get_terminal_size, + get_terminal_size: Callable[[], os.terminal_size] = os.get_terminal_size, get_time: Callable[[], float] = time.monotonic, is_tty: Optional[bool] = None, + color: Optional[bool] = None, + verbose: bool = False, + filter_padding: bool = False, ) -> None: #: Ordered dict of build ID -> info self.total = total @@ -756,15 +1200,60 @@ def __init__( self.tracked_build_id = "" # identifier of the package whose logs we follow self.search_term = "" self.search_mode = False + self.log_ends_with_newline = True + self.actual_jobs: int = 0 + self.target_jobs: int = 0 + self.blocked: bool = False self.stdout = stdout self.get_terminal_size = get_terminal_size + self.terminal_size = os.terminal_size((0, 0)) + self.terminal_size_changed: bool = True self.get_time = get_time - self.is_tty = is_tty if is_tty is not None else self.stdout.isatty() + self.is_tty = is_tty if is_tty is not None else stdout.isatty() + if color is not None: + self.color = color + else: + self.color = spack.llnl.util.tty.color.get_color_when(stdout) + #: Verbose mode only applies to non-TTY where we want to track a single build log. + self.verbose = verbose and not self.is_tty + self.filter_padding = filter_padding + #: When True, suppress all terminal output (process is in background). + #: Controlling code is responsible for modifying this variable based on process state + self.headless = False + + def on_resize(self) -> None: + """Refresh cached terminal size and trigger a redraw.""" + self.terminal_size_changed = True + self.dirty = True - def add_build(self, spec: spack.spec.Spec, explicit: bool, control_w_conn: Connection) -> None: + def add_build( + self, + spec: spack.spec.Spec, + explicit: bool, + control_w_conn: Optional[Connection] = None, + log_path: Optional[str] = None, + ) -> None: """Add a new build to the display and mark the display as dirty.""" - self.builds[spec.dag_hash()] = BuildInfo(spec, explicit, control_w_conn) + build_info = BuildInfo(spec, explicit, control_w_conn, log_path, int(self.get_time())) + self.builds[spec.dag_hash()] = build_info + self.dirty = True + # Track the new build's logs when we're not already following another build. This applies + # only in non-TTY verbose mode. + if self.verbose and not self.tracked_build_id and control_w_conn is not None: + self.tracked_build_id = spec.dag_hash() + try: + os.write(control_w_conn.fileno(), b"1") + except OSError: + pass + + def remove_build(self, build_id: str) -> None: + """Remove a build from the display (e.g. after a binary cache miss before retry).""" + self.builds.pop(build_id, None) + if self.tracked_build_id == build_id: + self.tracked_build_id = "" + if not self.overview_mode: + self.overview_mode = True self.dirty = True def toggle(self) -> None: @@ -772,13 +1261,18 @@ def toggle(self) -> None: if self.overview_mode: self.next() else: + if not self.log_ends_with_newline: + self.stdout.buffer.write(b"\n") + self.log_ends_with_newline = True self.active_area_rows = 0 self.search_term = "" self.search_mode = False self.overview_mode = True self.dirty = True try: - os.write(self.builds[self.tracked_build_id].control_w_conn.fileno(), b"0") + conn = self.builds[self.tracked_build_id].control_w_conn + if conn is not None: + os.write(conn.fileno(), b"0") except (KeyError, OSError): pass self.tracked_build_id = "" @@ -786,6 +1280,7 @@ def toggle(self) -> None: def search_input(self, input: str) -> None: """Handle keyboard input when in search mode""" if input in ("\r", "\n"): + self.log_ends_with_newline = False self.next(1) elif input == "\x1b": # Escape self.search_mode = False @@ -813,7 +1308,8 @@ def _get_next(self, direction: int) -> Optional[str]: matching = [ build_id for build_id, build in self.builds.items() - if build.finished_time is None and self._is_displayed(build) + if (build.finished_time is None or build.state == "failed") + and self._is_displayed(build) ] if not matching: return None @@ -839,21 +1335,58 @@ def next(self, direction: int = 1) -> None: # Stop following the previous and start following the new build. if self.tracked_build_id: try: - os.write(self.builds[self.tracked_build_id].control_w_conn.fileno(), b"0") + conn = self.builds[self.tracked_build_id].control_w_conn + if conn is not None: + os.write(conn.fileno(), b"0") except (KeyError, OSError): pass self.tracked_build_id = new_build_id - # Tell the user we're following new logs, and instruct the child to start sending them. - self.stdout.write( - f"\n==> Following logs of {new_build.name}" f"\033[0;36m@{new_build.version}\033[0m\n" + version_str = ( + f"\033[0;36m@{new_build.version}\033[0m" if self.color else f"@{new_build.version}" ) - self.stdout.flush() - try: - os.write(new_build.control_w_conn.fileno(), b"1") - except (KeyError, OSError): - pass + prefix = "" if self.log_ends_with_newline else "\n" + + if new_build.state == "failed": + # For failed builds, show the stored log summary instead of following live logs. + self.stdout.write(f"{prefix}==> Log summary of {new_build.name}{version_str}\n") + self.log_ends_with_newline = True + if new_build.log_summary: + self.stdout.write(new_build.log_summary) + if new_build.log_path: + if not new_build.log_summary: + self.stdout.write("No errors parsed from log, see full log: ") + else: + self.stdout.write("Full log: ") + self.stdout.write(f"{new_build.log_path}\n") + self.stdout.flush() + else: + # Tell the user we're following new logs, and instruct the child to start sending. + self.stdout.write(f"{prefix}==> Following logs of {new_build.name}{version_str}\n") + self.log_ends_with_newline = True + self.stdout.flush() + try: + conn = new_build.control_w_conn + if conn is not None: + os.write(conn.fileno(), b"1") + except (KeyError, OSError): + pass + + def set_blocked(self, blocked: bool) -> None: + """Set whether all pending builds are blocked by another Spack process.""" + if blocked == self.blocked: + return + self.blocked = blocked + self.dirty = True + + def set_jobs(self, actual: int, target: int) -> None: + """Set the actual and target number of jobs to run concurrently.""" + if actual == self.actual_jobs and target == self.target_jobs: + return + self.actual_jobs = actual + self.target_jobs = target + self.dirty = True def update_state(self, build_id: str, state: str) -> None: """Update the state of a package and mark the display as dirty.""" @@ -863,20 +1396,39 @@ def update_state(self, build_id: str, state: str) -> None: if state in ("finished", "failed"): self.completed += 1 - build_info.finished_time = self.get_time() + CLEANUP_TIMEOUT + now = self.get_time() + build_info.duration = now - build_info.start_time + build_info.finished_time = now + CLEANUP_TIMEOUT - if build_id == self.tracked_build_id and not self.overview_mode: - self.toggle() + # Stop tracking the finished build's logs. + if build_id == self.tracked_build_id: + if not self.overview_mode: + self.toggle() + if self.verbose: + self.tracked_build_id = "" self.dirty = True - # For non-TTY output, print state changes immediately without colors - if not self.is_tty: - self.stdout.write( - f"{build_info.hash} {build_info.name}@{build_info.version}: {state}\n" + # For non-TTY output, print state changes immediately + if not self.is_tty and not self.headless: + line = "".join( + self._generate_line_components(build_info, static=True, now=self.get_time()) ) + self.stdout.write(line + "\n") self.stdout.flush() + def parse_log_summary(self, build_id: str) -> None: + """Parse the build log for errors/warnings and store the summary.""" + build_info = self.builds[build_id] + if not build_info.log_path or not os.path.exists(build_info.log_path): + return + errors, warnings, tail_event = parse_log_events(build_info.log_path, tail=20) + events = [*errors, *warnings] + if tail_event is not None: + events.append(tail_event) + if events: + build_info.log_summary = make_log_context(events) + def update_progress(self, build_id: str, current: int, total: int) -> None: """Update the progress of a package and mark the display as dirty.""" percent = int((current / total) * 100) @@ -887,7 +1439,7 @@ def update_progress(self, build_id: str, current: int, total: int) -> None: def update(self, finalize: bool = False) -> None: """Redraw the interactive display.""" - if not self.is_tty or not self.overview_mode: + if self.headless or not self.is_tty or not self.overview_mode: return now = self.get_time() @@ -914,39 +1466,69 @@ def update(self, finalize: bool = False) -> None: del self.builds[build_id] self.dirty = True - if not self.dirty: + if not self.dirty and not finalize: return # Build the overview output in a buffer and print all at once to avoid flickering. buffer = io.StringIO() - # Move cursor up to the start of the display area + # Move cursor up to the start of the display area assuming the same terminal width. If the + # terminal resized, lines may have wrapped, and we should've moved up further. We do not + # try to track that (would require keeping track of each line's width). if self.active_area_rows > 0: - buffer.write(f"\033[{self.active_area_rows}F") - - max_width, max_height = self.get_terminal_size() + buffer.write(f"\033[{self.active_area_rows}A\r") - self.total_lines = 0 - total_finished = len(self.finished_builds) + if self.terminal_size_changed: + self.terminal_size = self.get_terminal_size() + self.terminal_size_changed = False + # After resize, active_area_rows is invalidated due to possible line wrapping. Set to + # 0 to force newlines instead of cursor movement. + self.active_area_rows = 0 + max_width, max_height = self.terminal_size # First flush the finished builds. These are "persisted" in terminal history. - for build in self.finished_builds: - self._render_build(build, buffer, max_width) - self.finished_builds.clear() + if self.finished_builds: + for build in self.finished_builds: + self._render_build(build, buffer, now=now) + self._println(buffer, force_newline=True) # should scroll the terminal + self.finished_builds.clear() + # Finished builds can span multiple lines, overlapping our "active area", invalidating + # active_area_rows. Set to 0 to force newlines instead of cursor movement. + self.active_area_rows = 0 # Then a header followed by the active builds. This is the "mutable" part of the display. - long_header_len = len( - f"Progress: {self.completed}/{self.total} /: filter v: logs n/p: next/prev" - ) - if long_header_len < max_width: - self._println( - buffer, - f"\033[1mProgress:\033[0m {self.completed}/{self.total}" - " \033[36m/\033[0m: filter \033[36mv\033[0m: logs" - " \033[36mn\033[0m/\033[36mp\033[0m: next/prev", + self.total_lines = 0 + + if not finalize: + if self.color: + bold = "\033[1m" + reset = "\033[0m" + cyan = "\033[36m" + else: + bold = reset = cyan = "" + + if self.actual_jobs != self.target_jobs: + jobs_str = f"{self.actual_jobs}=>{self.target_jobs}" + else: + jobs_str = str(self.target_jobs) + long_header_len = len( + f"Progress: {self.completed}/{self.total} +/-: {jobs_str} jobs" + " /: filter v: logs n/p: next/prev" ) - else: - self._println(buffer, f"\033[1mProgress:\033[0m {self.completed}/{self.total}") + if long_header_len < max_width: + self._println( + buffer, + f"{bold}Progress:{reset} {self.completed}/{self.total}" + f" {cyan}+{reset}/{cyan}-{reset}: " + f"{jobs_str} jobs" + f" {cyan}/{reset}: filter {cyan}v{reset}: logs" + f" {cyan}n{reset}/{cyan}p{reset}: next/prev", + ) + else: + self._println(buffer, f"{bold}Progress:{reset} {self.completed}/{self.total}") + + if self.blocked and not any(pkg.finished_time is None for pkg in self.builds.values()): + self._println(buffer, "Waiting for other Spack install process...") displayed_builds = ( [b for b in self.builds.values() if self._is_displayed(b)] @@ -963,7 +1545,8 @@ def update(self, finalize: bool = False) -> None: if i > truncate_at: self._println(buffer, f"{len_builds - i + 1} more...") break - self._render_build(build, buffer, max_width) + self._render_build(build, buffer, max_width, now=now) + self._println(buffer) if self.search_mode: buffer.write(f"filter> {self.search_term}\033[K") @@ -976,44 +1559,52 @@ def update(self, finalize: bool = False) -> None: self.stdout.flush() # Update the number of lines drawn for next time. It reflects the number of active builds. - self.active_area_rows = self.total_lines - total_finished + self.active_area_rows = self.total_lines self.dirty = False # Schedule next UI update self.next_update = now + SPINNER_INTERVAL / 2 - def _println(self, buffer: io.StringIO, line: str = "") -> None: + def _println(self, buffer: io.StringIO, line: str = "", force_newline: bool = False) -> None: """Print a line to the buffer, handling line clearing and cursor movement.""" self.total_lines += 1 if line: buffer.write(line) - if self.total_lines > self.active_area_rows: + if self.total_lines > self.active_area_rows or force_newline: buffer.write("\033[0m\033[K\n") # reset, clear to EOL, newline else: - buffer.write("\033[0m\033[K\033[1E") # reset, clear to EOL, move down 1 line + buffer.write("\033[0m\033[K\033[1B\r") # reset, clear to EOL, move to next line def print_logs(self, build_id: str, data: bytes) -> None: + if self.headless: + return # Discard logs we are not following. Generally this should not happen as we tell the child # to only send logs when we are following it. It could maybe happen while transitioning # between builds. - if self.overview_mode or build_id != self.tracked_build_id: + if build_id != self.tracked_build_id: return - # TODO: drop initial bytes from data until first newline (?) + if self.filter_padding: + data = padding_filter_bytes(data) self.stdout.buffer.write(data) self.stdout.flush() + self.log_ends_with_newline = data.endswith(b"\n") - def _render_build(self, build_info: BuildInfo, buffer: io.StringIO, max_width: int) -> None: + def _render_build( + self, build_info: BuildInfo, buffer: io.StringIO, max_width: int = 0, now: float = 0.0 + ) -> None: + """Print a single build line to the buffer, truncating to max_width (if > 0).""" line_width = 0 - for component in self._generate_line_components(build_info): + for component in self._generate_line_components(build_info, now=now): # ANSI escape sequence(s), does not contribute to width - if not component.startswith("\033"): + if not component.startswith("\033") and max_width > 0: line_width += len(component) if line_width > max_width: break buffer.write(component) - self._println(buffer) - def _generate_line_components(self, build_info: BuildInfo) -> Generator[str, None, None]: + def _generate_line_components( + self, build_info: BuildInfo, static: bool = False, now: float = 0.0 + ) -> Generator[str, None, None]: """Yield formatted line components for a package. Escape sequences are yielded as separate strings so they do not contribute to the line width.""" if build_info.external: @@ -1022,43 +1613,71 @@ def _generate_line_components(self, build_info: BuildInfo) -> Generator[str, Non indicator = "[+]" elif build_info.state == "failed": indicator = "[x]" + elif static: + indicator = "[ ]" else: indicator = f"[{self.spinner_chars[self.spinner_index]}]" - if build_info.state == "failed": - yield "\033[31m" # red - elif build_info.state == "finished": - yield "\033[32m" # green + if self.color: + if build_info.state == "failed": + yield "\033[31m" # red + elif build_info.state == "finished": + yield "\033[32m" # green yield indicator - yield "\033[0m" # reset + if self.color: + yield "\033[0m" # reset yield " " - yield "\033[0;90m" # dark gray + if self.color: + yield "\033[0;90m" # dark gray yield build_info.hash - yield "\033[0m" # reset + if self.color: + yield "\033[0m" # reset yield " " - # Package name in bold white if explicit, default otherwise + # Package name in bold if explicit, default otherwise if build_info.explicit: - yield "\033[1;37m" # bold white + if self.color: + yield "\033[1m" yield build_info.name - yield "\033[0m" # reset + if self.color: + yield "\033[0m" # reset else: yield build_info.name - yield "\033[0;36m" # cyan + if self.color: + yield "\033[0;36m" # cyan yield f"@{build_info.version}" - yield "\033[0m" # reset + if self.color: + yield "\033[0m" # reset # progress or state if build_info.progress_percent is not None: yield " fetching" yield f": {build_info.progress_percent}%" elif build_info.state == "finished": - yield f" {build_info.prefix}" + prefix = build_info.prefix + yield f" {padding_filter(prefix) if self.filter_padding else prefix}" + elif build_info.state == "failed": + yield " failed" + if build_info.log_path: + yield f": {build_info.log_path}" else: yield f" {build_info.state}" + # Duration + elapsed = ( + build_info.duration + if build_info.duration is not None + else (now - build_info.start_time) + ) + if elapsed > 0: + if self.color: + yield "\033[0;90m" # dark gray + yield f" ({pretty_duration(elapsed)})" + if self.color: + yield "\033[0m" + Nodes = Dict[str, spack.spec.Spec] Edges = Dict[str, Set[str]] @@ -1077,6 +1696,8 @@ def __init__( install_deps: bool, database: spack.database.Database, overwrite_set: Optional[Set[str]] = None, + tests: Union[bool, List[str], Set[str]] = False, + explicit_set: Optional[Set[str]] = None, ): """Construct a build graph from the given specs. This includes only packages that need to be installed. Installed packages are pruned from the graph, and build dependencies are only @@ -1086,11 +1707,16 @@ def __init__( self.parent_to_child: Dict[str, Set[str]] = {} self.child_to_parent: Dict[str, Set[str]] = {} overwrite_set = overwrite_set or set() - specs_to_prune: Set[str] = set() + explicit_set = explicit_set or set() + self.pruned: Set[str] = set() + self.done: Set[str] = set() + self.force_source: Set[str] = set() stack: List[Tuple[spack.spec.Spec, InstallPolicy]] = [ (s, root_policy) for s in self.nodes.values() ] + self.tests = tests + with database.read_transaction(): # Set the install prefix for each spec based on the db record or store layout for s in spack.traverse.traverse_nodes(specs): @@ -1105,16 +1731,18 @@ def __init__( spec, install_policy = stack.pop() key = spec.dag_hash() _, record = database.query_by_spec_hash(key) + depflag = self._base_deptypes(spec) - # Conditionally include build dependencies + # Conditionally include build dependencies. Don't prune installed specs + # that need to be marked explicit so they flow through the DB write path. if record and record.installed and key not in overwrite_set: - specs_to_prune.add(key) - dependencies = spec.dependencies(deptype=dt.LINK | dt.RUN) - elif install_policy == "cache_only" and not include_build_deps: - dependencies = spec.dependencies(deptype=dt.LINK | dt.RUN) - else: - dependencies = spec.dependencies(deptype=dt.BUILD | dt.LINK | dt.RUN) + # If it needs to be marked explicit, keep it in the graph (don't prune). + if key not in explicit_set or record.explicit: + self.pruned.add(key) + elif install_policy == "source_only" or include_build_deps: + depflag |= dt.BUILD + dependencies = spec.dependencies(deptype=depflag) self.parent_to_child[key] = {d.dag_hash() for d in dependencies} # Enqueue new dependencies @@ -1134,11 +1762,11 @@ def __init__( # If we're not installing the package itself, mark root specs for pruning too if not install_package: - specs_to_prune.update(s.dag_hash() for s in specs) + self.pruned.update(s.dag_hash() for s in specs) # Prune specs from the build graph. Their parents become parents of their children and # their children become children of their parents. - for key in specs_to_prune: + for key in self.pruned: for parent in self.child_to_parent.get(key, ()): self.parent_to_child[parent].remove(key) self.parent_to_child[parent].update(self.parent_to_child.get(key, ())) @@ -1149,6 +1777,14 @@ def __init__( self.child_to_parent.pop(key, None) self.nodes.pop(key, None) + # Check that all prefixes to be created are unique. + prefixes = [s.prefix for s in self.nodes.values() if not s.external] + if len(prefixes) != len(set(prefixes)): + raise spack.error.InstallError( + "Install prefix collision: " + + ", ".join(p for p in prefixes if prefixes.count(p) > 1) + ) + # If we're not installing dependencies, verify that all remaining nodes in the build graph # after pruning are roots. If there are any non-root nodes, it means there are uninstalled # dependencies that we're not supposed to install. @@ -1160,6 +1796,15 @@ def __init__( "installed" ) + def _base_deptypes(self, spec: spack.spec.Spec) -> dt.DepFlag: + """Returns the dependency types that are always eagerly traversed. These are LINK, RUN, and + conditionally TEST, but excludes BUILD. Build deps are deferred until after a build cache + miss.""" + deptypes = dt.LINK | dt.RUN + if self.tests is True or (self.tests and spec.name in self.tests): + deptypes |= dt.TEST + return deptypes + def enqueue_parents(self, dag_hash: str, pending_builds: List[str]) -> None: """After a spec is installed, remove it from the graph and enqueue any parents that are now ready to install. @@ -1168,6 +1813,7 @@ def enqueue_parents(self, dag_hash: str, pending_builds: List[str]) -> None: dag_hash: The dag_hash of the spec that was just installed pending_builds: List to append parent specs that are ready to build """ + self.done.add(dag_hash) # Remove node and edges from the node in the build graph self.parent_to_child.pop(dag_hash, None) self.nodes.pop(dag_hash, None) @@ -1183,8 +1829,534 @@ def enqueue_parents(self, dag_hash: str, pending_builds: List[str]) -> None: if not children: pending_builds.append(parent) + def has_unexpanded_build_deps(self, dag_hash: str) -> bool: + return bool(self.get_unexpanded_build_deps(dag_hash)) + + def get_unexpanded_build_deps(self, dag_hash: str) -> List["spack.spec.Spec"]: + """Returns a list of unprocessed build deps for a spec.""" + spec = self.nodes[dag_hash] + base_deptypes = self._base_deptypes(spec) + unexpanded = [] + for edge in spec.edges_to_dependencies(depflag=dt.BUILD): + if (edge.depflag & base_deptypes) == 0: + unexpanded.append(edge.spec) + return unexpanded + + def expand_build_deps( + self, + spec_hashes: List[str], + pending_builds: List[str], + database: "spack.database.Database", + dependencies_policy: InstallPolicy = "auto", + ) -> List[str]: + """Expand build dependencies for a list of specs after binary cache misses. + + Adds the spec's build deps and their transitive runtime deps to the graph. When + ``dependencies_policy`` is ``"source_only"``, build deps of newly added specs are included + immediately. Installed deps are skipped without adding edges. + + The caller must hold the database read lock and have called ``db._read()``. + + Returns the list of newly added dag hashes.""" + # Seed with tuples of (parent_hash, dep) for each to-be-expanded build dep + stack = [(h, dep) for h in spec_hashes for dep in self.get_unexpanded_build_deps(h)] + newly_added: List[str] = [] + + while stack: + parent_hash, dep = stack.pop() + dep_hash = dep.dag_hash() + + # Skip installed deps + if dep_hash in self.pruned or dep_hash in self.done: + continue + + # If already in the graph (e.g. overwrite build in progress), add edge but don't + # re-add node. This must be checked before the DB installed check, because an + # overwrite build is installed in the DB but not yet done. + if dep_hash in self.nodes: + self.parent_to_child.setdefault(parent_hash, set()).add(dep_hash) + self.child_to_parent.setdefault(dep_hash, set()).add(parent_hash) + continue + + _, record = database.query_by_spec_hash(dep_hash) + if record and record.installed: + self.done.add(dep_hash) + continue + + # Add forward/reverse edge + self.parent_to_child.setdefault(parent_hash, set()).add(dep_hash) + self.child_to_parent.setdefault(dep_hash, set()).add(parent_hash) + + # New node: add to graph and recurse into its link/run/test deps + self.nodes[dep_hash] = dep + self.parent_to_child.setdefault(dep_hash, set()) + newly_added.append(dep_hash) + + deptype = self._base_deptypes(dep) + if dependencies_policy == "source_only": + deptype |= dt.BUILD + for child in dep.dependencies(deptype=deptype): + stack.append((dep_hash, child)) + + # Enqueue nodes that are ready (no uninstalled children) + for h in newly_added: + if not self.parent_to_child[h]: + pending_builds.append(h) + for dag_hash in spec_hashes: + if not self.parent_to_child[dag_hash]: + pending_builds.append(dag_hash) + + return newly_added + + +class ScheduleResult(NamedTuple): + """Return value of :func:`schedule_builds`.""" + + #: True if any pending builds were blocked on locks held by other processes. + blocked: bool + #: ``(dag_hash, lock)`` pairs where the write lock is held and the caller must start the build + #: and eventually release the lock. + to_start: List[Tuple[str, spack.util.lock.Lock]] + #: ``(dag_hash, spec, lock)`` triples found already installed by another process; the read lock + #: is held and the caller must add it to retained_read_locks. + newly_installed: List[Tuple[str, spack.spec.Spec, spack.util.lock.Lock]] + #: Actions to mark already installed specs explicit in the DB. + to_mark_explicit: List[MarkExplicitAction] + + +def schedule_builds( + pending: List[str], + build_graph: BuildGraph, + db: spack.database.Database, + prefix_locker: spack.database.SpecLocker, + overwrite: Set[str], + overwrite_time: float, + capacity: int, + needs_jobserver_token: bool, + jobserver: JobServer, + explicit: Set[str], +) -> ScheduleResult: + """Try to schedule as many pending builds as possible. + + For each pending spec, attempts to acquire a non-blocking per-spec write lock. If the write + lock times out, a read lock is tried as a fallback: a successful read lock means the first + process finished and downgraded its write lock. If the DB confirms the spec is installed, it + is captured as newly_installed; if the DB says it is not installed, the concurrent process was + likely killed mid-build, and the spec is retried next iteration. Under both the DB read lock + and the prefix lock, checks whether another process has already installed the spec. If so, + captures it as newly_installed (caller enqueues parents) and keeps a read lock on the prefix + to prevent concurrent uninstall. Otherwise, acquires a jobserver token if needed and adds the + (dag_hash, lock) pair to to_start (caller launches the build). + + Args: + pending: List of dag hashes pending installation; modified in-place. + build_graph: The build dependency graph; used for node lookup and parent enqueueing. + db: Package database; used for read lock and installed-status queries. + prefix_locker: Per-spec write locker. + overwrite: Set of dag hashes to overwrite even if already installed. + overwrite_time: Timestamp (from time.time()) at which the overwrite install was requested. + A spec in ``overwrite`` whose DB installation_time >= overwrite_time was installed by + a concurrent process after our request started and should be treated as done. + capacity: Maximum number of new builds to add to to_start in this call. + needs_jobserver_token: True if a jobserver token is required for the first new build. + jobserver: Jobserver for acquiring tokens. + explicit: Set of dag hashes to mark explicit in the DB if found already installed. + + Returns: + A :class:`ScheduleResult` with ``blocked``, ``to_start``, and ``newly_installed`` + fields; see :class:`ScheduleResult` for field semantics. + """ + to_start: List[Tuple[str, spack.util.lock.Lock]] = [] + newly_installed: List[Tuple[str, spack.spec.Spec, spack.util.lock.Lock]] = [] + to_mark_explicit: List[MarkExplicitAction] = [] + blocked = True + + # Acquire the DB read lock non-blocking; hold it throughout the loop so the in-memory snapshot + # stays consistent while we acquire per-spec prefix locks. + if not db.lock.try_acquire_read(): + return ScheduleResult(blocked, to_start, newly_installed, to_mark_explicit) + + try: + db._read() # refresh in-memory snapshot under the read lock + + idx = 0 + while capacity and idx < len(pending): + dag_hash = pending[idx] + spec = build_graph.nodes[dag_hash] + lock = prefix_locker.lock(spec) + + if lock.try_acquire_write(): + blocked = False + have_write = True + elif lock.try_acquire_read(): + have_write = False + else: + idx += 1 + continue + + # Check installed status under the DB read lock and prefix lock. + upstream, record = db.query_by_spec_hash(dag_hash) + + # If the spec is already installed, treat it as done regardless of lock type. + # A spec in the overwrite set is also treated as done if another process installed it + # after our overwrite request was created (installation_time >= overwrite_time). + if ( + record + and record.installed + and (dag_hash not in overwrite or record.installation_time >= overwrite_time) + ): + if have_write: + lock.downgrade_write_to_read() + # keep the read lock (either downgraded or already a read lock) + del pending[idx] + newly_installed.append((dag_hash, spec, lock)) + # It's already installed, but needs to be marked as explicitly installed in the DB. + if dag_hash in explicit and not record.explicit: + to_mark_explicit.append(MarkExplicitAction(spec)) + build_graph.enqueue_parents(dag_hash, pending) + continue + + if not have_write: + # If have to install but only got a read lock, try it in next iteration of the + # event loop. + lock.release_read() + idx += 1 + continue + + # Write lock acquired: proceed with scheduling. + # Don't schedule builds for specs from upstream databases. + if upstream and record and not record.installed: + lock.release_write() + raise spack.error.InstallError( + f"Cannot install {spec}: it is uninstalled in an upstream database." + ) + + # Defensively assert prefix invariants + if not spec.external: + if ( + dag_hash in overwrite + and record + and record.installed + and record.path != spec.prefix + ): + # Cannot do an overwrite install to a different prefix. + lock.release_write() + raise spack.error.InstallError( + f"Prefix mismatch in overwrite of {spec}: expected {record.path}, " + f"got {spec.prefix}" + ) + elif dag_hash not in overwrite and spec.prefix in db._installed_prefixes: + # Prevent install prefix collision with other specs. + lock.release_write() + raise spack.error.InstallError( + f"Cannot install {spec}: prefix {spec.prefix} already exists" + ) + + # Acquire a jobserver token if needed. The first (implicit) job needs no token. + if needs_jobserver_token and not jobserver.acquire(1): + lock.release_write() + break # no tokens available right now; stop scheduling + + del pending[idx] + to_start.append((dag_hash, lock)) + capacity -= 1 + needs_jobserver_token = True # all subsequent jobs need a token + + finally: + db.lock.release_read() + + return ScheduleResult(blocked, to_start, newly_installed, to_mark_explicit) + + +def _node_to_roots(roots: List[spack.spec.Spec]) -> Dict[str, FrozenSet[str]]: + """Map each node in a graph to the set of root node DAG hashes that can reach it. + + Args: + roots: List of root specs. + + Returns: + A dictionary mapping each node's dag_hash to a frozenset of root dag_hashes. + """ + node_to_roots: Dict[str, FrozenSet[str]] = { + s.dag_hash(): frozenset([s.dag_hash()]) for s in roots + } + + for edge in spack.traverse.traverse_edges( + roots, order="topo", cover="edges", root=False, key=spack.traverse.by_dag_hash + ): + parent_roots = node_to_roots[edge.parent.dag_hash()] + child_hash = edge.spec.dag_hash() + existing = node_to_roots.get(child_hash) + + if existing is None: + node_to_roots[child_hash] = parent_roots # keep a reference if no mutation is needed + elif not parent_roots.issubset(existing): + node_to_roots[child_hash] = existing | parent_roots + + return node_to_roots + + +class ReportData: + """Data collected for reports during installation.""" + + def __init__(self, roots: List[spack.spec.Spec]): + self.roots = roots + self.build_records: Dict[str, spack.report.InstallRecord] = {} + + def start_record(self, spec: spack.spec.Spec) -> None: + """Begin an InstallRecord for a spec that is about to be built.""" + if spec.external: + return + record = spack.report.InstallRecord(spec) + record.start() + self.build_records[spec.dag_hash()] = record + + def finish_record( + self, spec: spack.spec.Spec, exitcode: int, log_path: Optional[str] = None + ) -> None: + """Mark the InstallRecord for a spec as succeeded or failed.""" + record = self.build_records.get(spec.dag_hash()) + if record is None or spec.external: + return + if exitcode == ExitCode.SUCCESS: + record.succeed(log_path) + else: + record.fail( + spack.error.InstallError( + f"Installation of {spec.name} failed; see log for details" + ), + log_path, + ) + + def finalize( + self, reports: Dict[str, spack.report.RequestRecord], build_graph: BuildGraph + ) -> None: + """Finalize InstallRecords and append them to RequestRecords after all builds finish. + + Args: + reports: Map of root dag_hash to RequestRecord to append to. + build_graph: The build graph containing all nodes and their states. + """ + node_to_roots = _node_to_roots(self.roots) + + for spec in spack.traverse.traverse_nodes(self.roots): + h = spec.dag_hash() + if h in self.build_records: + record = self.build_records[h] + else: + record = spack.report.InstallRecord(spec) + if spec.external: + msg = "Spec is external" + elif h in build_graph.pruned: + msg = "Spec was not scheduled for installation" + elif h in build_graph.nodes: + msg = "Dependencies failed to install" + else: + # If not installed or failed (build_records), not statically pruned ahead of + # time (build_graph.pruned), and also not scheduled (build_graph.nodes), it + # means it was in pending_builds or running_builds but never started/finished. + # This branch is followed on KeyboardInterrupt and --fail-fast. + msg = "Installation was interrupted" + record.skip(msg=msg) + + for root_hash in node_to_roots[h]: + reports[root_hash].append_record(record) + + +class NullReportData(ReportData): + """No-op drop-in for ReportData when no reporter is configured. + + Avoids creating InstallRecords and reading log files on every completed build.""" + + def __init__(self) -> None: + pass + + def start_record(self, spec: spack.spec.Spec) -> None: + pass + + def finish_record( + self, spec: spack.spec.Spec, exitcode: int, log_path: Optional[str] = None + ) -> None: + pass + + def finalize( + self, reports: Dict[str, spack.report.RequestRecord], build_graph: "BuildGraph" + ) -> None: + pass + + +class TerminalState: + """Manages terminal settings, stdin selector registration, and suspend/resume signals. + + Installs a SIGTSTP handler that restores the terminal before suspending and re-applies it + on resume. After waking up it checks whether the process is in the foreground or background + and enables or suppresses interactive output accordingly. + + Optional ``on_suspend`` / ``on_resume`` hooks are called just before the process suspends + and just after it wakes, allowing callers to pause and resume child processes.""" + + def __init__( + self, + selector: selectors.BaseSelector, + build_status: BuildStatus, + on_suspend: Optional[Callable[[], None]] = None, + on_resume: Optional[Callable[[], None]] = None, + ) -> None: + self.selector = selector + self.build_status = build_status + self.on_suspend = on_suspend + self.on_resume = on_resume + self.old_stdin_settings = termios.tcgetattr(sys.stdin) + self.sigwinch_r = -1 + self.sigwinch_w = -1 + + def setup(self) -> None: + """Set cbreak mode, register stdin and signal pipes in the selector.""" + + # SIGWINCH self-pipe (stdout must be a tty too) + if sys.stdout.isatty(): + self.sigwinch_r, self.sigwinch_w = os.pipe() + os.set_blocking(self.sigwinch_r, False) + os.set_blocking(self.sigwinch_w, False) + self.selector.register(self.sigwinch_r, selectors.EVENT_READ, "sigwinch") + self.old_sigwinch = signal.signal(signal.SIGWINCH, self._handle_sigwinch) + else: + self.old_sigwinch = None + + self.old_sigtstp = signal.signal(signal.SIGTSTP, self._handle_sigtstp) + + # Start correctly depending on whether we're foregrounded or backgrounded + self.build_status.headless = True + if not _is_background_tty(sys.stdin): + self.enter_foreground() + + def teardown(self) -> None: + """Restore terminal settings and signal handlers, close pipes.""" + with ignore_signal(signal.SIGTTOU): + termios.tcsetattr(sys.stdin, termios.TCSADRAIN, self.old_stdin_settings) + + for sig, old in ((signal.SIGTSTP, self.old_sigtstp), (signal.SIGWINCH, self.old_sigwinch)): + if old is not None: + try: + signal.signal(sig, old) + except Exception as e: + spack.llnl.util.tty.debug(f"Failed to restore signal handler for {sig}: {e}") + + if sys.stdin.fileno() in self.selector.get_map(): + self.selector.unregister(sys.stdin.fileno()) + + for fd in (self.sigwinch_r, self.sigwinch_w): + if fd < 0: + continue + if fd in self.selector.get_map(): + self.selector.unregister(fd) + try: + os.close(fd) + except Exception as e: + spack.llnl.util.tty.debug(f"Failed to close sigwinch pipe {fd}: {e}") + + def _handle_sigtstp(self, signum: int, frame: object) -> None: + """Restore terminal before suspending, then re-install handler after resume.""" + + # Reset so the first redraw after resume doesn't overwrite the shell's + # prompt / "$ fg" line. + self.build_status.active_area_rows = 0 + + # Restore terminal so the user's shell works normally while we're stopped. + with ignore_signal(signal.SIGTTOU): + termios.tcsetattr(sys.stdin, termios.TCSANOW, self.old_stdin_settings) + + # Force headless mode before suspending so that enter_foreground() doesn't + # exit early when we resume, ensuring terminal settings are re-applied. + self.build_status.headless = True + + # Actually suspend: reset to default handler then re-send SIGTSTP. + if self.on_suspend is not None: + self.on_suspend() + signal.signal(signal.SIGTSTP, signal.SIG_DFL) + os.kill(os.getpid(), signal.SIGTSTP) + + # Execution resumes here after SIGCONT. Re-install our handler. + signal.signal(signal.SIGTSTP, self._handle_sigtstp) + + if self.on_resume is not None: + self.on_resume() + self.handle_continue() + + def _handle_sigwinch(self, signum: int, frame: object) -> None: + try: + os.write(self.sigwinch_w, b"\x00") + except OSError: + pass + + def enter_foreground(self) -> None: + """Restore interactive terminal mode.""" + if not self.build_status.headless: + return + + # We save old settings right before applying cbreak. + # If we started in the background, bash may have had the terminal in its own + # readline (raw) mode when __init__ ran. Waiting until we are foregrounded + # ensures we capture the shell's exported 'sane' configuration for this job. + self.old_stdin_settings = termios.tcgetattr(sys.stdin) + + with ignore_signal(signal.SIGTTOU): + tty.setcbreak(sys.stdin.fileno()) + + if sys.stdin.fileno() not in self.selector.get_map(): + self.selector.register(sys.stdin.fileno(), selectors.EVENT_READ, "stdin") + self.build_status.headless = False + self.build_status.dirty = True + + def enter_background(self) -> None: + """Suppress output and stop reading stdin to avoid SIGTTIN/SIGTTOU.""" + if sys.stdin.fileno() in self.selector.get_map(): + self.selector.unregister(sys.stdin.fileno()) + self.build_status.headless = True + + def handle_continue(self) -> None: + """Detect whether the process is in the foreground or background and adjust accordingly.""" + if _is_background_tty(sys.stdin): + self.enter_background() + else: + self.enter_foreground() + + +def _signal_children(running_builds: Dict[int, ChildInfo], sig: signal.Signals) -> None: + """Send a signal to the process group of each running build.""" + for child in running_builds.values(): + try: + pid = child.proc.pid + if pid is not None: + os.killpg(pid, sig) + except OSError: + pass + + +class StdinReader: + """Helper class to do non-blocking, incremental decoding of stdin, stripping ANSI escape + sequences. The input is the backing file descriptor for stdin (instead of the TextIOWrapper) to + avoid double buffering issues: the event loop triggers when the fd is ready to read, and if we + do a partial read from the TextIOWrapper, it will likely drain the fd and buffer the remainder + internally, which the event loop is not aware of, and user input doesn't come through.""" + + def __init__(self, fd: int) -> None: + self.fd = fd + #: Handle multi-byte UTF-8 characters + self.decoder = codecs.getincrementaldecoder("utf-8")(errors="replace") + #: For stripping out arrow and navigation keys + self.ansi_escape_re = re.compile(r"\x1b\[[0-9;]*[A-Za-z~]") + + def read(self) -> str: + try: + chars = self.decoder.decode(os.read(self.fd, 1024)) + return self.ansi_escape_re.sub("", chars) + except OSError: + return "" + class PackageInstaller: + explicit: Set[str] def __init__( self, @@ -1211,35 +2383,45 @@ def __init__( concurrent_packages: Optional[int] = None, root_policy: InstallPolicy = "auto", dependencies_policy: InstallPolicy = "auto", + create_reports: bool = False, ) -> None: assert install_package or install_deps, "Must install package, dependencies or both" - if fail_fast: - raise NotImplementedError("Fail-fast installs are not implemented") - elif fake: - raise NotImplementedError("Fake installs are not implemented") - elif install_source: - raise NotImplementedError("Installing sources is not implemented") - elif stop_at is not None: - raise NotImplementedError("Stopping at an install phase is not implemented") - elif stop_before is not None: - raise NotImplementedError("Stopping before an install phase is not implemented") - elif tests is not False: - raise NotImplementedError("Tests during install are not implemented") - # verbose and concurrent_packages are not worth erroring out for + self.install_source = install_source + self.stop_at = stop_at + self.stop_before = stop_before + self.tests: Union[bool, List[str], Set[str]] = tests + + self.db = spack.store.STORE.db specs = [pkg.spec for pkg in packages] + # No point trying cache when there are no binary mirrors configured. + if not spack.mirrors.mirror.MirrorCollection(binary=True): + if root_policy == "auto": + root_policy = "source_only" + if dependencies_policy == "auto": + dependencies_policy = "source_only" self.root_policy: InstallPolicy = root_policy self.dependencies_policy: InstallPolicy = dependencies_policy self.include_build_deps = include_build_deps #: Set of DAG hashes to overwrite (if already installed) self.overwrite: Set[str] = set(overwrite) if overwrite else set() + #: Time at which the overwrite install was requested; used to detect concurrent overwrites. + self.overwrite_time: float = time.time() self.keep_prefix = keep_prefix + self.fail_fast = fail_fast # Buffer for incoming, partially received state data from child processes self.state_buffers: Dict[int, str] = {} + if explicit is True: + self.explicit = {spec.dag_hash() for spec in specs} + elif explicit is False: + self.explicit = set() + else: + self.explicit = explicit + # Build the dependency graph self.build_graph = BuildGraph( specs, @@ -1248,18 +2430,25 @@ def __init__( include_build_deps, install_package, install_deps, - spack.store.STORE.db, + self.db, self.overwrite, + tests, + self.explicit, ) #: check what specs we could fetch from binaries (checks against cache, not remotely) - spack.binary_distribution.BINARY_INDEX.update() + try: + spack.binary_distribution.BINARY_INDEX.update() + except spack.binary_distribution.FetchCacheError: + pass + self.binary_cache_for_spec = { s.dag_hash(): spack.binary_distribution.BINARY_INDEX.find_by_hash(s.dag_hash()) for s in self.build_graph.nodes.values() } self.unsigned = unsigned self.dirty = dirty + self.fake = fake self.restage = restage self.keep_stage = keep_stage self.skip_patch = skip_patch @@ -1269,72 +2458,117 @@ def __init__( parent for parent, children in self.build_graph.parent_to_child.items() if not children ] - if explicit is True: - self.explicit = {spec.dag_hash() for spec in specs} - elif explicit is False: - self.explicit = set() - else: - self.explicit = explicit + #: specs awaiting build-dep expansion (deferred until DB read lock is available) + self.pending_expansions: List[str] = [] + self.verbose = verbose self.running_builds: Dict[int, ChildInfo] = {} - self.build_status = BuildStatus(len(self.build_graph.nodes)) + self.log_paths: Dict[str, str] = {} + self.build_status = BuildStatus( + len(self.build_graph.nodes), + verbose=verbose, + filter_padding=spack.store.STORE.has_padding(), + ) self.jobs = spack.config.determine_number_of_jobs(parallel=True) - self.reports: Dict[str, spack.report.RequestRecord] = {} + self.build_status.actual_jobs = self.jobs + self.build_status.target_jobs = self.jobs + if concurrent_packages is None: + concurrent_packages_config = spack.config.get("config:concurrent_packages", 0) + # The value 0 in config means no limit (other than self.jobs) + if concurrent_packages_config == 0: + self.capacity = sys.maxsize + else: + self.capacity = concurrent_packages_config + else: + self.capacity = concurrent_packages + + # The reports property is what the old installer has and used as public interface. + if create_reports: + self.reports = {spec.dag_hash(): spack.report.RequestRecord(spec) for spec in specs} + self.report_data = ReportData(specs) + else: + self.reports = {} + self.report_data = NullReportData() + + self.next_database_write = 0.0 def install(self) -> None: - # This installer has not implemented the per-spec exclusive locks during installation. - # Instead, take an exclusive lock on the entire range to avoid that other Spack install - # process start installing the same specs. - lock = spack.util.lock.Lock( - str(spack.store.STORE.prefix_locker.lock_path), desc="prefix lock" - ) - lock.acquire_write() - try: - self._installer() - finally: - lock.release_write() + self._installer() def _installer(self) -> None: + spack.store.STORE.install_sbang() jobserver = JobServer(self.jobs) - - # Set stdin to non-blocking for key press detection - if sys.stdin.isatty(): - old_stdin_settings = termios.tcgetattr(sys.stdin) - tty.setcbreak(sys.stdin.fileno()) - else: - old_stdin_settings = None - selector = selectors.DefaultSelector() - selector.register(sys.stdin.fileno(), selectors.EVENT_READ, "stdin") - # Setup the database write lock. TODO: clean this up - if isinstance(spack.store.STORE.db.lock, spack.util.lock.Lock): - spack.store.STORE.db.lock._ensure_parent_directory() - spack.store.STORE.db.lock._file = spack.llnl.util.lock.FILE_TRACKER.get_fh( - spack.store.STORE.db.lock.path + # Set up terminal handling (cbreak, signals, stdin registration) + terminal: Optional[TerminalState] = None + stdin_reader: Optional[StdinReader] = None + if sys.stdin.isatty(): + stdin_reader = StdinReader(sys.stdin.fileno()) + terminal = TerminalState( + selector, + self.build_status, + on_suspend=lambda: _signal_children(self.running_builds, signal.SIGSTOP), + on_resume=lambda: _signal_children(self.running_builds, signal.SIGCONT), ) + terminal.setup() + + # Finished builds that have not yet been written to the database. + database_actions: List[DatabaseAction] = [] + # Prefix read locks retained after DB flush (downgraded from write locks in _save_to_db). + retained_read_locks: List[spack.util.lock.Lock] = [] - to_insert_in_database: List[ChildInfo] = [] failures: List[spack.spec.Spec] = [] + finished_pids: List[int] = [] try: - # Start the first job immediately, as it does not require a jobserver token. - if self.pending_builds and not self.running_builds: - self._start(selector, jobserver) - - while self.pending_builds or self.running_builds or to_insert_in_database: - # Only monitor the jobserver if we have pending builds. - if self.pending_builds and jobserver.r not in selector.get_map(): + # Try to schedule builds immediately. The first job does not require a token. + if self.pending_builds: + blocked = self._schedule_builds( + selector, jobserver, retained_read_locks, database_actions + ) + self.build_status.set_blocked(blocked and not self.running_builds) + + while ( + self.pending_builds + or self.running_builds + or database_actions + or self.pending_expansions + ): + # Monitor the jobserver when we have pending builds, capacity, and at least one + # spec is not locked by another process. Also listen if the target parallelism is + # reduced. + wake_on_jobserver = ( + self.pending_builds + and self.capacity + and not blocked + or not jobserver.has_target_parallelism() + ) + if wake_on_jobserver and jobserver.r not in selector.get_map(): selector.register(jobserver.r, selectors.EVENT_READ, "jobserver") - elif not self.pending_builds and jobserver.r in selector.get_map(): + elif not wake_on_jobserver and jobserver.r in selector.get_map(): selector.unregister(jobserver.r) - jobserver_token_available = False stdin_ready = False - events = selector.select(timeout=SPINNER_INTERVAL) + if self.build_status.headless: + # no UI to update, but check background to foreground transition periodically + timeout = HEADLESS_WAKE_INTERVAL + elif self.build_status.is_tty: + timeout = SPINNER_INTERVAL + else: + # when not in interactive mode, wake least often (no spinner/terminal updates) + timeout = DATABASE_WRITE_INTERVAL + events = selector.select(timeout=timeout) + + finished_pids.clear() - finished_pids = [] + # The transition "suspended to foreground/background" is handled in the signal + # handler, but there's no SIGCONT event in the transition of background to + # foreground, so we conditionally poll for that here (headless case). In the + # headless case the event loop only fires once per second, so this is cheap enough. + if terminal and self.build_status.headless and not _is_background_tty(sys.stdin): + terminal.enter_foreground() for key, _ in events: data = key.data @@ -1347,123 +2581,366 @@ def _installer(self) -> None: self._handle_child_state(key.fd, child_info, selector) elif data.name == "sentinel": finished_pids.append(data.pid) - elif data == "jobserver": - jobserver_token_available = True elif data == "stdin": stdin_ready = True - + elif data == "sigwinch": + assert terminal is not None + os.read(terminal.sigwinch_r, 64) # drain the pipe + self.build_status.on_resize() + elif data == "jobserver" and not jobserver.has_target_parallelism(): + jobserver.maybe_discard_tokens() + self.build_status.set_jobs(jobserver.num_jobs, jobserver.target_jobs) + + current_time = time.monotonic() for pid in finished_pids: - build = self.running_builds.pop(pid) - jobserver.release() - build.cleanup(selector) - if build.proc.exitcode == 0: - to_insert_in_database.append(build) - self.build_status.update_state(build.spec.dag_hash(), "finished") - else: - failures.append(build.spec) - self.build_status.update_state(build.spec.dag_hash(), "failed") - - if stdin_ready: - try: - char = sys.stdin.read(1) - except OSError: - continue - overview = self.build_status.overview_mode - if overview and self.build_status.search_mode: - self.build_status.search_input(char) - elif overview and char == "/": - self.build_status.enter_search() - elif char == "v" or char in ("q", "\x1b") and not overview: - self.build_status.toggle() - elif char == "n": - self.build_status.next(1) - elif char == "p" or char == "N": - self.build_status.next(-1) - - # Flush installed packages to the database and enqueue any parents that are now - # ready. - if to_insert_in_database and self._save_to_db(to_insert_in_database): - for entry in to_insert_in_database: - self.build_graph.enqueue_parents( - entry.spec.dag_hash(), self.pending_builds - ) - to_insert_in_database.clear() - - # Again, the first job should start immediately and does not require a token. - if self.pending_builds and not self.running_builds: - self._start(selector, jobserver) - - # For the rest we try to obtain tokens from the jobserver. - if self.pending_builds and jobserver_token_available: - # Then we try to schedule as many jobs as we can acquire tokens for. - max_new_jobs = len(self.pending_builds) - for _ in range(jobserver.acquire(max_new_jobs)): - self._start(selector, jobserver) + self._handle_finished_build( + pid, current_time, jobserver, selector, failures, database_actions + ) + + if failures and self.fail_fast: + # Terminate other builds to actually fail fast. We continue in the event loop + # waiting for child processes to finish, which may take a little while. + for child in self.running_builds.values(): + child.proc.terminate() + self.pending_builds.clear() + + if stdin_ready and stdin_reader is not None: + for char in stdin_reader.read(): + overview = self.build_status.overview_mode + if overview and self.build_status.search_mode: + self.build_status.search_input(char) + elif overview and char == "/": + self.build_status.enter_search() + elif char == "v" or char in ("q", "\x1b") and not overview: + self.build_status.toggle() + elif char == "n": + self.build_status.next(1) + elif char == "p" or char == "N": + self.build_status.next(-1) + elif char == "+": + jobserver.increase_parallelism() + self.build_status.set_jobs(jobserver.num_jobs, jobserver.target_jobs) + elif char == "-": + jobserver.decrease_parallelism() + self.build_status.set_jobs(jobserver.num_jobs, jobserver.target_jobs) + + # Insert into the database if we have any finished builds, and either the delay + # interval has passed, or we're done with all builds. The database save is not + # guaranteed; it fails if another process holds the lock. We'll try again next + # iteration of the event loop in that case. + if ( + database_actions + and ( + current_time >= self.next_database_write + or not (self.pending_builds or self.running_builds) + ) + and self._save_to_db(database_actions, retained_read_locks) + ): + database_actions.clear() + + # Try to expand build deps for cache-miss specs. This requires a read lock on the + # database, meaning that it can take several iterations of the event loop in case + # of contention with other processes. + if self.pending_expansions: + self._try_expand_build_deps() + + # Try to schedule more builds, acquiring per-spec locks and jobserver tokens. + if self.capacity and self.pending_builds: + blocked = self._schedule_builds( + selector, jobserver, retained_read_locks, database_actions + ) + self.build_status.set_blocked(blocked and not self.running_builds) # Finally update the UI self.build_status.update() - except KeyboardInterrupt: - # Cleanup running builds. - for child in self.running_builds.values(): - child.proc.join() - raise finally: - # Restore terminal settings - if old_stdin_settings: - termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_stdin_settings) + # First ensure that the user's terminal state is restored. + if terminal is not None: + terminal.teardown() + + # Flush any not-yet-written successful builds to the DB; save the exception on error + # to be re-raised after best-effort cleanup. + db_exc = None + if database_actions: + try: + with self.db.write_transaction(): + for action in database_actions: + action.save_to_db(self.db) + except Exception as e: + db_exc = e + + # Send SIGTERM to running builds; this is a no-op in the successful case. + for child in self.running_builds.values(): + try: + child.proc.terminate() + except Exception: + pass + + # Release our jobserver token for each terminated build and then join. + for child in self.running_builds.values(): + try: + jobserver.release() + child.proc.join(timeout=30) + if child.proc.is_alive(): + child.proc.kill() + child.proc.join() + except Exception: + pass + + # Release all held locks best-effort, so that one failure does not prevent the others + # from being released. + for child in self.running_builds.values(): + child.release_prefix_lock() - # Clean up resources - # Final cleanup of any remaining finished packages before exit - self.build_status.overview_mode = True - self.build_status.update(finalize=True) - selector.close() - jobserver.close() + for lock in retained_read_locks: + try: + lock.release_read() + except Exception: + pass + for action in database_actions: + action.release_prefix_lock() + + try: + self.build_status.overview_mode = True + self.build_status.update(finalize=True) + selector.close() + jobserver.close() + except Exception: + pass + + # Re-raise the DB exception if any. + if db_exc is not None: + raise db_exc + + try: + self.report_data.finalize(self.reports, build_graph=self.build_graph) + except Exception as e: + spack.llnl.util.tty.debug(f"[{__name__}]: Failed to finalize reports: {e}]") + + # Clean up temp log files of successful builds now that reports have consumed them. + if not self.keep_stage: + failed_hashes = {s.dag_hash() for s in failures} + for dag_hash, log_path in self.log_paths.items(): + if log_path == os.devnull or dag_hash in failed_hashes: + continue + try: + os.unlink(log_path) + except OSError: + pass if failures: - lines = [f"{s}: {s.package.log_path}" for s in failures] + for s in failures: + build_info = self.build_status.builds[s.dag_hash()] + if build_info and build_info.log_summary: + sys.stderr.write(build_info.log_summary) + lines = [f"{s}: {self.log_paths[s.dag_hash()]}" for s in failures] raise spack.error.InstallError( "The following packages failed to install:\n" + "\n".join(lines) ) - def _save_to_db(self, to_insert_in_database: List[ChildInfo]) -> bool: - db = spack.store.STORE.db + def _handle_finished_build( + self, + pid: int, + current_time: float, + jobserver: JobServer, + selector: selectors.BaseSelector, + failures: List[spack.spec.Spec], + database_actions: List[DatabaseAction], + ) -> None: + """Handle a build that has finished. Remove from running_builds; release jobserver token; + update UI state; defer database insertion if successful; possibly reschedule if failed with + cache miss; register failures.""" + build = self.running_builds.pop(pid) + dag_hash = build.spec.dag_hash() + self.capacity += 1 + jobserver.release() + self.build_status.set_jobs(jobserver.num_jobs, jobserver.target_jobs) + self._drain_child_output(build, selector) + self._drain_child_state(build, selector) + exitcode = build.close(selector) + self.report_data.finish_record(build.spec, exitcode, build.log_path) + + if exitcode == ExitCode.SUCCESS: + # Schedule successful builds for batched database insertion. We don't release the + # prefix lock here; that strictly happens after a successful db write. + database_actions.append(build) + self.build_graph.enqueue_parents(dag_hash, self.pending_builds) + self.next_database_write = current_time + DATABASE_WRITE_INTERVAL + self.build_status.update_state(dag_hash, "finished") + return + + # When we don't have to do a db write, we can release the lock immediately. + build.release_prefix_lock() + + is_root = dag_hash in self.build_graph.roots + user_policy = self.root_policy if is_root else self.dependencies_policy + + if exitcode == ExitCode.STOPPED_AT_PHASE: + return # the user requested early stopping; don't treat as failure + elif exitcode == ExitCode.BUILD_CACHE_MISS and user_policy == "auto": + # Check if we can reschedule this as a source build after a build cache miss. If so, + # return early without recording a failure. + self.build_graph.force_source.add(dag_hash) + self.build_status.remove_build(dag_hash) + if self.build_graph.has_unexpanded_build_deps(dag_hash): + self.pending_expansions.append(dag_hash) + else: + self.pending_builds.append(dag_hash) + elif not failures or not self.fail_fast: + # Record a failure. In fail-fast mode, only record the first failure; subsequent + # failures may be a consequence of us terminating other builds. + failures.append(build.spec) + self.build_status.update_state(dag_hash, "failed") + self.build_status.parse_log_summary(dag_hash) + + def _try_expand_build_deps(self) -> None: + """Try to expand build deps for specs with cache misses. Non-blocking: returns immediately + if the DB read lock is unavailable.""" + if not self.db.lock.try_acquire_read(): + return try: - # Only try to get the lock once (non-blocking). If it fails, try it next time. - if db.lock.acquire_write(timeout=1e-9): - db._read() - except spack.util.lock.LockTimeoutError: + self.db._read() + newly_added = self.build_graph.expand_build_deps( + self.pending_expansions, self.pending_builds, self.db, self.dependencies_policy + ) + for h in newly_added: + self.binary_cache_for_spec[h] = ( + spack.binary_distribution.BINARY_INDEX.find_by_hash(h) + ) + self.build_status.total += len(newly_added) + self.pending_expansions.clear() + finally: + self.db.lock.release_read() + + def _save_to_db( + self, + database_actions: List[DatabaseAction], + retained_read_locks: List[spack.util.lock.Lock], + ) -> bool: + if not self.db.lock.try_acquire_write(): return False try: - for entry in to_insert_in_database: - db._add(entry.spec, explicit=entry.explicit) - return True + self.db._read() + for action in database_actions: + action.save_to_db(self.db) finally: - db.lock.release_write(db._write) - - def _start(self, selector: selectors.BaseSelector, jobserver: JobServer) -> None: - dag_hash = self.pending_builds.pop() + self.db.lock.release_write(self.db._write) + + # DB has been written and flushed; downgrade per-spec prefix write locks to read locks so + # other processes can see the specs are installed, while preventing concurrent uninstalls. + for action in database_actions: + if action.prefix_lock is not None: + try: + action.prefix_lock.downgrade_write_to_read() + retained_read_locks.append(action.prefix_lock) + except Exception: + action.prefix_lock.release_write() + raise + finally: + action.prefix_lock = None + + return True + + def _schedule_builds( + self, + selector: selectors.BaseSelector, + jobserver: JobServer, + retained_read_locks: List[spack.util.lock.Lock], + database_actions: List[DatabaseAction], + ) -> bool: + """Try to schedule as many pending builds as possible. + + Delegates to the module-level schedule_builds() function and then performs the + side-effects that require the selector and running-build state: updating build_status for + specs that were found already installed, and launching new builds via _start(). + + Preconditions: self.capacity > 0 and self.pending_builds is not empty. + + Returns True if we had capacity to schedule, but were blocked by locks held by other + processes. In that case we should not monitor the jobserver for new tokens, since we'd end + up in a busy wait loop until the locks are released. + """ + result = schedule_builds( + pending=self.pending_builds, + build_graph=self.build_graph, + db=self.db, + prefix_locker=spack.store.STORE.prefix_locker, + overwrite=self.overwrite, + overwrite_time=self.overwrite_time, + capacity=self.capacity, + needs_jobserver_token=bool(self.running_builds), + jobserver=jobserver, + explicit=self.explicit, + ) + blocked = result.blocked + database_actions.extend(result.to_mark_explicit) + # Specs installed by another process. + for dag_hash, spec, lock in result.newly_installed: + retained_read_locks.append(lock) + explicit = dag_hash in self.explicit + self.build_status.add_build(spec, explicit=explicit) + self.build_status.update_state(dag_hash, "finished") + # Specs we can start building ourselves. + for dag_hash, lock in result.to_start: + self._start(selector, jobserver, dag_hash, lock) + return blocked + + def _install_policy(self, dag_hash: str, is_root: bool) -> InstallPolicy: + if dag_hash in self.build_graph.force_source: + return "source_only" + policy = self.root_policy if is_root else self.dependencies_policy + if policy == "auto" and not self.include_build_deps: + return "cache_only" + return policy + + def _start( + self, + selector: selectors.BaseSelector, + jobserver: JobServer, + dag_hash: str, + prefix_lock: spack.util.lock.Lock, + ) -> None: + self.capacity -= 1 explicit = dag_hash in self.explicit spec = self.build_graph.nodes[dag_hash] is_develop = spec.is_develop + tests = self.tests + run_tests = tests is True or bool(tests and spec.name in tests) + is_root = dag_hash in self.build_graph.roots + # Both possible sub-processes (cache install, source build) append to the same log file. + if dag_hash not in self.log_paths: + if spec.external: + self.log_paths[dag_hash] = os.devnull + else: + log_fd, log_path = tempfile.mkstemp( + prefix=f"spack-stage-{spec.name}-{spec.version}-{spec.dag_hash()}-", + suffix=".log", + dir=spack.stage.get_stage_root(), + ) + os.close(log_fd) + self.log_paths[dag_hash] = log_path + child_info = start_build( spec, explicit=explicit, mirrors=self.binary_cache_for_spec[dag_hash], unsigned=self.unsigned, - install_policy=( - self.root_policy - if dag_hash in self.build_graph.roots - else self.dependencies_policy - ), + install_policy=self._install_policy(dag_hash, is_root), dirty=self.dirty, # keep_stage/restage logic taken from installer.py keep_stage=self.keep_stage or is_develop, restage=self.restage and not is_develop, - overwrite=dag_hash in self.overwrite, keep_prefix=self.keep_prefix, skip_patch=self.skip_patch, + fake=self.fake, + install_source=self.install_source, + run_tests=run_tests, jobserver=jobserver, + log_path=self.log_paths[dag_hash], + stop_before=self.stop_before if is_root else None, + stop_at=self.stop_at if is_root else None, ) + child_info.prefix_lock = prefix_lock pid = child_info.proc.pid assert type(pid) is int self.running_builds[pid] = child_info @@ -1475,8 +2952,12 @@ def _start(self, selector: selectors.BaseSelector, jobserver: JobServer) -> None ) selector.register(child_info.proc.sentinel, selectors.EVENT_READ, FdInfo(pid, "sentinel")) self.build_status.add_build( - child_info.spec, explicit=explicit, control_w_conn=child_info.control_w_conn + child_info.spec, + explicit=explicit, + control_w_conn=child_info.control_w_conn, + log_path=child_info.log_path, ) + self.report_data.start_record(spec) def _handle_child_logs( self, r_fd: int, child_info: ChildInfo, selector: selectors.BaseSelector @@ -1486,6 +2967,8 @@ def _handle_child_logs( # There might be more data than OUTPUT_BUFFER_SIZE, but we will read that in the next # iteration of the event loop to keep things responsive. data = os.read(r_fd, OUTPUT_BUFFER_SIZE) + except BlockingIOError: + return except OSError: data = None @@ -1498,6 +2981,18 @@ def _handle_child_logs( self.build_status.print_logs(child_info.spec.dag_hash(), data) + def _drain_child_output(self, child_info: ChildInfo, selector: selectors.BaseSelector) -> None: + """Read and print any remaining output from a finished child's pipe.""" + r_fd = child_info.output_r_conn.fileno() + while r_fd in selector.get_map(): + self._handle_child_logs(r_fd, child_info, selector) + + def _drain_child_state(self, child_info: ChildInfo, selector: selectors.BaseSelector) -> None: + """Read and process any remaining state messages from a finished child's pipe.""" + r_fd = child_info.state_r_conn.fileno() + while r_fd in selector.get_map(): + self._handle_child_state(r_fd, child_info, selector) + def _handle_child_state( self, r_fd: int, child_info: ChildInfo, selector: selectors.BaseSelector ) -> None: @@ -1506,6 +3001,8 @@ def _handle_child_state( # There might be more data than OUTPUT_BUFFER_SIZE, but we will read that in the next # iteration of the event loop to keep things responsive. data = os.read(r_fd, OUTPUT_BUFFER_SIZE) + except BlockingIOError: + return except OSError: data = None @@ -1528,10 +3025,19 @@ def _handle_child_state( for line in lines: if not line: continue - message = json.loads(line) + try: + message = json.loads(line) + except json.JSONDecodeError: + continue if "state" in message: self.build_status.update_state(child_info.spec.dag_hash(), message["state"]) elif "progress" in message and "total" in message: self.build_status.update_progress( child_info.spec.dag_hash(), message["progress"], message["total"] ) + elif "installed_from_binary_cache" in message: + child_info.spec.package.installed_from_binary_cache = True + + +class BinaryCacheMiss(spack.error.SpackError): + pass diff --git a/lib/spack/spack/oci/oci.py b/lib/spack/spack/oci/oci.py index 0f4c4bb2d13ee5..0bc95bcf94b917 100644 --- a/lib/spack/spack/oci/oci.py +++ b/lib/spack/spack/oci/oci.py @@ -7,7 +7,6 @@ import os import urllib.error import urllib.parse -from http.client import HTTPResponse from typing import List, NamedTuple, Tuple from urllib.request import Request @@ -59,12 +58,12 @@ def list_tags(ref: ImageReference, _urlopen: spack.oci.opener.MaybeOpen = None) while True: # Fetch tags request = Request(url=fetch_url) - response = _urlopen(request) - spack.oci.opener.ensure_status(request, response, 200) - tags.update(json.load(response)["tags"]) + with _urlopen(request) as response: + spack.oci.opener.ensure_status(request, response, 200) + tags.update(json.load(response)["tags"]) - # Check for pagination - link_header = response.headers["Link"] + # Check for pagination + link_header = response.headers["Link"] if link_header is None: break @@ -141,20 +140,20 @@ def upload_blob( url=ref.uploads_url(), method="POST", headers={"Content-Length": "0"} ) - response = _urlopen(request) + with _urlopen(request) as response: + # Created the blob in one go. + if response.status == 201: + return True - # Created the blob in one go. - if response.status == 201: - return True + # Otherwise, do another PUT request. + spack.oci.opener.ensure_status(request, response, 202) + assert "Location" in response.headers - # Otherwise, do another PUT request. - spack.oci.opener.ensure_status(request, response, 202) - assert "Location" in response.headers + # Can be absolute or relative, joining handles both + upload_url = with_query_param( + ref.endpoint(response.headers["Location"]), "digest", str(digest) + ) - # Can be absolute or relative, joining handles both - upload_url = with_query_param( - ref.endpoint(response.headers["Location"]), "digest", str(digest) - ) f.seek(0) request = Request( @@ -164,9 +163,8 @@ def upload_blob( headers={"Content-Type": "application/octet-stream", "Content-Length": str(file_size)}, ) - response = _urlopen(request) - - spack.oci.opener.ensure_status(request, response, 201) + with _urlopen(request) as response: + spack.oci.opener.ensure_status(request, response, 201) return True @@ -205,9 +203,8 @@ def upload_manifest( headers={"Content-Type": manifest["mediaType"]}, ) - response = _urlopen(request) - - spack.oci.opener.ensure_status(request, response, 201) + with _urlopen(request) as response: + spack.oci.opener.ensure_status(request, response, 201) return digest, size @@ -222,8 +219,8 @@ def blob_exists( """Checks if a blob exists in an OCI registry""" try: _urlopen = _urlopen or spack.oci.opener.urlopen - response = _urlopen(Request(url=ref.blob_url(digest), method="HEAD")) - return response.status == 200 + with _urlopen(Request(url=ref.blob_url(digest), method="HEAD")) as response: + return response.status == 200 except urllib.error.HTTPError as e: if e.getcode() == 404: return False @@ -314,34 +311,33 @@ def get_manifest_and_config( _urlopen = _urlopen or spack.oci.opener.urlopen # Get manifest - response: HTTPResponse = _urlopen( + with _urlopen( Request(url=ref.manifest_url(), headers={"Accept": ", ".join(all_content_type)}) - ) - - # Recurse when we find an index - if response.headers["Content-Type"] in index_content_type: - if recurse == 0: - raise Exception("Maximum recursion depth reached while fetching OCI manifest") - - index = json.load(response) - manifest_meta = next( - manifest - for manifest in index["manifests"] - if manifest["platform"]["architecture"] == architecture - ) + ) as response: + # Recurse when we find an index + if response.headers["Content-Type"] in index_content_type: + if recurse == 0: + raise Exception("Maximum recursion depth reached while fetching OCI manifest") + + index = json.load(response) + manifest_meta = next( + manifest + for manifest in index["manifests"] + if manifest["platform"]["architecture"] == architecture + ) - return get_manifest_and_config( - ref.with_digest(manifest_meta["digest"]), - architecture=architecture, - recurse=recurse - 1, - _urlopen=_urlopen, - ) + return get_manifest_and_config( + ref.with_digest(manifest_meta["digest"]), + architecture=architecture, + recurse=recurse - 1, + _urlopen=_urlopen, + ) - # Otherwise, require a manifest - if response.headers["Content-Type"] not in manifest_content_type: - raise Exception(f"Unknown content type {response.headers['Content-Type']}") + # Otherwise, require a manifest + if response.headers["Content-Type"] not in manifest_content_type: + raise Exception(f"Unknown content type {response.headers['Content-Type']}") - manifest = json.load(response) + manifest = json.load(response) # Download, verify and cache config file config_digest = Digest.from_string(manifest["config"]["digest"]) diff --git a/lib/spack/spack/oci/opener.py b/lib/spack/spack/oci/opener.py index 76e7c1a6cc023a..1e983d8c6f72a3 100644 --- a/lib/spack/spack/oci/opener.py +++ b/lib/spack/spack/oci/opener.py @@ -7,8 +7,6 @@ import base64 import json import re -import socket -import time import urllib.error import urllib.parse import urllib.request @@ -433,31 +431,4 @@ def ensure_status(request: urllib.request.Request, response: HTTPResponse, statu ) -def default_retry(f, retries: int = 5, sleep=None): - sleep = sleep or time.sleep - - def wrapper(*args, **kwargs): - for i in range(retries): - try: - return f(*args, **kwargs) - except OSError as e: - # Retry on internal server errors, and rate limit errors - # Potentially this could take into account the Retry-After header - # if registries support it - if i + 1 != retries and ( - ( - isinstance(e, urllib.error.HTTPError) - and (500 <= e.code < 600 or e.code == 429) - ) - or ( - isinstance(e, urllib.error.URLError) - and isinstance(e.reason, socket.timeout) - ) - or isinstance(e, socket.timeout) - ): - # Exponential backoff - sleep(2**i) - continue - raise - - return wrapper +default_retry = spack.util.web.retry_on_transient_error diff --git a/lib/spack/spack/operating_systems/windows_os.py b/lib/spack/spack/operating_systems/windows_os.py index c9f32efc46c666..e965991f1c63d7 100755 --- a/lib/spack/spack/operating_systems/windows_os.py +++ b/lib/spack/spack/operating_systems/windows_os.py @@ -21,7 +21,7 @@ def windows_version(): # include the build number as this provides important information # for low lever packages and components like the SDK and WDK # The build number is the version component that would otherwise - # be the patch version in sematic versioning, i.e. z of x.y.z + # be the patch version in semantic versioning, i.e. z of x.y.z return Version(platform.version()) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 80a0c240e0d327..cd361702c83d3b 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -132,10 +132,9 @@ from spack.url import substitute_version as substitute_version_in_url from spack.user_environment import environment_modifications_for_specs from spack.util.elf import delete_needed_from_elf, delete_rpath, get_elf_compat, parse_elf -from spack.util.environment import EnvironmentModifications +from spack.util.environment import EnvironmentModifications, set_env from spack.util.environment import filter_system_paths as _filter_system_paths from spack.util.environment import is_system_path as _is_system_path -from spack.util.environment import set_env from spack.util.executable import Executable, ProcessError, which, which_string from spack.util.filesystem import fix_darwin_install_name from spack.util.libc import libc_from_dynamic_linker, parse_dynamic_linker diff --git a/lib/spack/spack/package_base.py b/lib/spack/spack/package_base.py index a53aa39aff3f06..586bd6ff02d430 100644 --- a/lib/spack/spack/package_base.py +++ b/lib/spack/spack/package_base.py @@ -303,7 +303,7 @@ def __new__(cls, name, bases, attr_dict): def on_package_attributes(**attr_dict): - """Decorator: executes instance function only if object has attr valuses. + """Decorator: executes instance function only if object has attr values. Executes the decorated method only if at the moment of calling the instance has attributes that are equal to certain values. @@ -320,9 +320,7 @@ def _wrapper(instance, *args, **kwargs): has_all_attributes = all([hasattr(instance, key) for key in attr_dict]) if has_all_attributes: has_the_right_values = all( - [ - getattr(instance, key) == value for key, value in attr_dict.items() - ] # NOQA: ignore=E501 + [getattr(instance, key) == value for key, value in attr_dict.items()] # NOQA: ignore=E501 ) if has_the_right_values: func(instance, *args, **kwargs) @@ -338,6 +336,8 @@ class PackageViewMixin: overriding these functions. """ + spec: spack.spec.Spec + def view_source(self): """The source root directory that will be added to the view: files are added such that their path relative to the view destination matches @@ -410,7 +410,7 @@ def _by_subkey( """Convert a dict of dicts keyed by when/subkey into a dict of lists keyed by subkey. Optional Arguments: - when: if ``True``, don't discared the ``when`` specs; return a 2-level dictionary + when: if ``True``, don't discard the ``when`` specs; return a 2-level dictionary keyed by subkey and when spec. """ # very hard to define this type to be conditional on `when` @@ -547,7 +547,7 @@ class PackageBase(WindowsRPath, PackageViewMixin, metaclass=PackageMeta): compiler = DeprecatedCompiler() #: Class level dictionary populated by :func:`~spack.directives.version` directives - versions: dict + versions: Dict[StandardVersion, Dict[str, Any]] #: Class level dictionary populated by :func:`~spack.directives.resource` directives resources: Dict[spack.spec.Spec, List[Resource]] #: Class level dictionary populated by :func:`~spack.directives.depends_on` and @@ -922,7 +922,7 @@ def keep_werror(self) -> Optional[Literal["all", "specific", "none"]]: def version(self): if not self.spec.versions.concrete: raise ValueError( - "Version requested for a package that" " does not have a concrete version." + "Version requested for a package that does not have a concrete version." ) return self.spec.versions[0] @@ -1209,7 +1209,10 @@ def _make_stages(self) -> Tuple[stg.StageComposite, List[stg.Stage]]: link_format = spack.config.get("config:develop_stage_link") if not link_format: link_format = "build-{arch}-{hash:7}" - stage_link = self.spec.format_path(link_format) + if link_format == "None": + stage_link = None + else: + stage_link = self.spec.format_path(link_format) source_stage = stg.DevelopStage( stg.compute_stage_name(self.spec), dev_path, stage_link ) @@ -1464,7 +1467,7 @@ def provides(self, vpkg_name: str) -> bool: def intersects(self, spec: spack.spec.Spec) -> bool: """Context-ful intersection that takes into account package information. - By design, ``Spec.intersects()`` does not know anything about package metdata. + By design, ``Spec.intersects()`` does not know anything about package metadata. This avoids unnecessary package lookups and keeps things efficient where extra information is not needed, and it decouples ``Spec`` from ``PackageBase``. @@ -1626,8 +1629,9 @@ def do_fetch(self, mirror_only=False): deprecated = spack.config.get("config:deprecated") if not deprecated and self.versions.get(self.version, {}).get("deprecated", False): tty.warn( - "{0} is deprecated and may be removed in a future Spack " - "release.".format(self.spec.format("{name}{@version}")) + "{0} is deprecated and may be removed in a future Spack release.".format( + self.spec.format("{name}{@version}") + ) ) # Ask the user whether to install deprecated version if we're @@ -2039,8 +2043,8 @@ def do_test(self, *, dirty=False, externals=False, timeout: Optional[int] = None self.tester.stand_alone_tests(kwargs, timeout=timeout) - def unit_test_check(self) -> bool: - """Hook for unit tests to assert things about package internals. + def _unit_test_check(self) -> bool: + """Hook for Spack's own unit tests to assert things about package internals. Unit tests can override this function to perform checks after ``Package.install`` and all post-install hooks run, but before @@ -2332,7 +2336,7 @@ def rpath(self): # Do not include Windows system libraries in the rpath interface # these libraries are handled automatically by VS/VCVARS and adding # Spack derived system libs into the link path or address space of a program - # can result in conflicting versions, which makes Spack packages less useable + # can result in conflicting versions, which makes Spack packages less usable if sys.platform == "win32": rpaths = [self.prefix.bin] rpaths.extend( diff --git a/lib/spack/spack/patch.py b/lib/spack/spack/patch.py index 8f9c5decd41e05..4893c9a0c28223 100644 --- a/lib/spack/spack/patch.py +++ b/lib/spack/spack/patch.py @@ -6,7 +6,7 @@ import os import pathlib import sys -from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Type, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Set, Tuple, Type, Union import spack import spack.error @@ -420,7 +420,9 @@ def to_json(self, stream: Any) -> None: """ sjson.dump({"patches": self.index}, stream) - def patch_for_package(self, sha256: str, pkg: "spack.package_base.PackageBase") -> Patch: + def patch_for_package( + self, sha256: str, pkg: Type["spack.package_base.PackageBase"], *, validate: bool = False + ) -> Patch: """Look up a patch in the index and build a patch object for it. We build patch objects lazily because building them requires that @@ -428,7 +430,9 @@ def patch_for_package(self, sha256: str, pkg: "spack.package_base.PackageBase") Args: sha256: sha256 hash to look up - pkg: Package object to get patch for. + pkg: Package class to get patch for. + validate: if True, validate the cached entry against the owner's current package + class and raise ``PatchLookupError`` if the entry is missing or stale. Returns: The patch object. @@ -449,42 +453,64 @@ def patch_for_package(self, sha256: str, pkg: "spack.package_base.PackageBase") f"Couldn't find patch for package {pkg.fullname} with sha256: {sha256}" ) + if validate: + # Validate the cached entry against the owner's current package class + owner = patch_dict.get("owner") + if not owner: + raise spack.error.PatchLookupError( + f"Patch for {pkg.fullname} with sha256 {sha256} has no owner in cache" + ) + try: + owner_pkg_cls = self.repository.get_pkg_class(owner) + current_index = PatchCache._index_patches(owner_pkg_cls, self.repository) + except Exception as e: + raise spack.error.PatchLookupError( + f"Could not validate patch cache for {pkg.fullname}: {e}" + ) from e + current_sha_index = current_index.get(sha256) + if not current_sha_index or current_sha_index.get(fullname) != patch_dict: + raise spack.error.PatchLookupError( + f"Stale patch cache entry for {pkg.fullname} with sha256: {sha256}" + ) + # add the sha256 back (we take it out on write to save space, # because it's the index key) patch_dict = dict(patch_dict) patch_dict["sha256"] = sha256 return from_dict(patch_dict, repository=self.repository) - def update_package(self, pkg_fullname: str) -> None: + def update_packages(self, pkgs_fullname: Set[str]) -> None: """Update the patch cache. Args: pkg_fullname: package to update. """ # remove this package from any patch entries that reference it. - empty = [] - for sha256, package_to_patch in self.index.items(): - remove = [] - for fullname, patch_dict in package_to_patch.items(): - if patch_dict["owner"] == pkg_fullname: - remove.append(fullname) + if self.index: + empty = [] + for sha256, package_to_patch in self.index.items(): + remove = [] + for fullname, patch_dict in package_to_patch.items(): + if patch_dict["owner"] in pkgs_fullname: + remove.append(fullname) - for fullname in remove: - package_to_patch.pop(fullname) + for fullname in remove: + package_to_patch.pop(fullname) - if not package_to_patch: - empty.append(sha256) + if not package_to_patch: + empty.append(sha256) - # remove any entries that are now empty - for sha256 in empty: - del self.index[sha256] + # remove any entries that are now empty + for sha256 in empty: + del self.index[sha256] # update the index with per-package patch indexes - pkg_cls = self.repository.get_pkg_class(pkg_fullname) - partial_index = self._index_patches(pkg_cls, self.repository) - for sha256, package_to_patch in partial_index.items(): - p2p = self.index.setdefault(sha256, {}) - p2p.update(package_to_patch) + for pkg_fullname in pkgs_fullname: + pkg_cls = self.repository.get_pkg_class(pkg_fullname) + partial_index = self._index_patches(pkg_cls, self.repository) + for sha256, package_to_patch in partial_index.items(): + p2p = self.index.setdefault(sha256, {}) + p2p.update(package_to_patch) def update(self, other: "PatchCache") -> None: """Update this cache with the contents of another. @@ -518,6 +544,9 @@ def _index_patches( patch_dict.pop("sha256") # save some space index[patch.sha256] = {pkg_class.fullname: patch_dict} + if not pkg_class._patches_dependencies: + return index + for deps_by_name in pkg_class.dependencies.values(): for dependency in deps_by_name.values(): for patch_list in dependency.patches.values(): diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index bfede02ea60ce4..707dc60184ac91 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -8,6 +8,7 @@ throughout Spack and should bring in a minimal number of external dependencies. """ + import os from pathlib import PurePath diff --git a/lib/spack/spack/platforms/_platform.py b/lib/spack/spack/platforms/_platform.py index 3aa493cd068246..d0c1f189495a4e 100644 --- a/lib/spack/spack/platforms/_platform.py +++ b/lib/spack/spack/platforms/_platform.py @@ -83,7 +83,7 @@ def operating_system(self, name): def setup_platform_environment(self, pkg, env): """Platform-specific build environment modifications. - This method is meant toi be overridden by subclasses, when needed. + This method is meant to be overridden by subclasses, when needed. """ pass diff --git a/lib/spack/spack/provider_index.py b/lib/spack/spack/provider_index.py index 886a89c0cb0831..674014543099ce 100644 --- a/lib/spack/spack/provider_index.py +++ b/lib/spack/spack/provider_index.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Classes and functions to manage providers of virtual dependencies""" + from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Set, Union import spack.error @@ -53,18 +54,8 @@ def __init__( self.repository = repository self.restrict = restrict self.providers = {} - - specs = specs or [] - for spec in specs: - if isinstance(spec, str): - from spack.spec import Spec - - spec = Spec(spec) - - if self.repository.is_virtual_safe(spec.name): - continue - - self.update(spec) + if specs: + self.update_packages(specs) def providers_for(self, virtual: Union[str, "spack.spec.Spec"]) -> List["spack.spec.Spec"]: """Return a list of specs of all packages that provide virtual packages with the supplied @@ -122,57 +113,55 @@ def __str__(self): def __repr__(self): return repr(self.providers) - def update(self, spec: Union[str, "spack.spec.Spec"]) -> None: + def update_packages(self, specs: Iterable[Union[str, "spack.spec.Spec"]]): """Update the provider index with additional virtual specs. Args: spec: spec potentially providing additional virtual specs """ - if isinstance(spec, str): - from spack.spec import Spec - - spec = Spec(spec) + from spack.spec import Spec - if not spec.name: - # Empty specs do not have a package - return + for spec in specs: + if not isinstance(spec, Spec): + spec = Spec(spec) - msg = "cannot update an index passing the virtual spec '{}'".format(spec.name) - assert not self.repository.is_virtual_safe(spec.name), msg + if not spec.name or self.repository.is_virtual_safe(spec.name): + # Only non-virtual packages with name can provide virtual specs. + continue - pkg_cls = self.repository.get_pkg_class(spec.name) - for provider_spec_readonly, provided_specs in pkg_cls.provided.items(): - for provided_spec in provided_specs: - # TODO: fix this comment. - # We want satisfaction other than flags - provider_spec = provider_spec_readonly.copy() - provider_spec.compiler_flags = spec.compiler_flags.copy() + pkg_cls = self.repository.get_pkg_class(spec.name) + for provider_spec_readonly, provided_specs in pkg_cls.provided.items(): + for provided_spec in provided_specs: + # TODO: fix this comment. + # We want satisfaction other than flags + provider_spec = provider_spec_readonly.copy() + provider_spec.compiler_flags = spec.compiler_flags.copy() - if spec.intersects(provider_spec, deps=False): - provided_name = provided_spec.name + if spec.intersects(provider_spec, deps=False): + provided_name = provided_spec.name - provider_map = self.providers.setdefault(provided_name, {}) - if provided_spec not in provider_map: - provider_map[provided_spec] = set() + provider_map = self.providers.setdefault(provided_name, {}) + if provided_spec not in provider_map: + provider_map[provided_spec] = set() - if self.restrict: - provider_set = provider_map[provided_spec] + if self.restrict: + provider_set = provider_map[provided_spec] - # If this package existed in the index before, - # need to take the old versions out, as they're - # now more constrained. - old = set([s for s in provider_set if s.name == spec.name]) - provider_set.difference_update(old) + # If this package existed in the index before, + # need to take the old versions out, as they're + # now more constrained. + old = {s for s in provider_set if s.name == spec.name} + provider_set.difference_update(old) - # Now add the new version. - provider_set.add(spec) + # Now add the new version. + provider_set.add(spec) - else: - # Before putting the spec in the map, constrain - # it so that it provides what was asked for. - constrained = spec.copy() - constrained.constrain(provider_spec) - provider_map[provided_spec].add(constrained) + else: + # Before putting the spec in the map, constrain + # it so that it provides what was asked for. + constrained = spec.copy() + constrained.constrain(provider_spec) + provider_map[provided_spec].add(constrained) def to_json(self, stream=None): """Dump a JSON representation of this object. @@ -207,14 +196,14 @@ def merge(self, other): spdict[provided_spec] = spdict[provided_spec].union(opdict[provided_spec]) - def remove_provider(self, pkg_name): + def remove_providers(self, pkg_names: Set[str]): """Remove a provider from the ProviderIndex.""" empty_pkg_dict = [] for pkg, pkg_dict in self.providers.items(): empty_pset = [] for provided, pset in pkg_dict.items(): - same_name = set(p for p in pset if p.fullname == pkg_name) - pset.difference_update(same_name) + to_remove = {spec for spec in pset if spec.name in pkg_names} + pset.difference_update(to_remove) if not pset: empty_pset.append(provided) diff --git a/lib/spack/spack/relocate.py b/lib/spack/spack/relocate.py index 0f42b191ff1f4f..bc61e3a20521f8 100644 --- a/lib/spack/spack/relocate.py +++ b/lib/spack/spack/relocate.py @@ -197,7 +197,7 @@ def _set_elf_rpaths_and_interpreter( def relocate_macho_binaries(path_names, prefix_to_prefix): """ - Use macholib python package to get the rpaths, depedent libraries + Use macholib python package to get the rpaths, dependent libraries and library identity for libraries from the MachO object. Modify them with the replacement paths queried from the dictionary mapping old layout prefixes to hashes and the dictionary mapping hashes to the new layout @@ -419,8 +419,9 @@ def fixup_macos_rpaths(spec): if not os.path.exists(prefix): raise RuntimeError( - "Could not fix up install prefix spec {0} because it does " - "not exist: {1!s}".format(prefix, spec.name) + "Could not fix up install prefix spec {0} because it does not exist: {1!s}".format( + prefix, spec.name + ) ) # Explore the installation prefix of the spec diff --git a/lib/spack/spack/repo.py b/lib/spack/spack/repo.py index b6e1687cfc8a92..c62be55213511a 100644 --- a/lib/spack/spack/repo.py +++ b/lib/spack/spack/repo.py @@ -11,6 +11,7 @@ import importlib.machinery import importlib.util import itertools +import math import os import re import shutil @@ -34,6 +35,7 @@ Tuple, Type, Union, + cast, ) import spack @@ -42,7 +44,6 @@ import spack.error import spack.llnl.path import spack.llnl.util.filesystem as fs -import spack.llnl.util.lang import spack.llnl.util.tty as tty import spack.patch import spack.paths @@ -57,6 +58,7 @@ import spack.util.path import spack.util.spack_yaml as syaml from spack.llnl.util.filesystem import working_dir +from spack.llnl.util.lang import Singleton, memoized if TYPE_CHECKING: import spack.package_base @@ -103,6 +105,25 @@ def namespace_from_fullname(fullname: str) -> str: return fullname +def name_from_fullname(fullname: str) -> str: + """Return the package name for the full module name. + + For instance:: + + name_from_fullname("spack.pkg.builtin.hdf5") == "hdf5" + name_from_fullname("spack_repo.x.y.z.packages.pkg_name.package") == "pkg_name" + + Args: + fullname: full name for the Python module + """ + if fullname.startswith(PKG_MODULE_PREFIX_V1): + _, _, pkg_module = fullname.rpartition(".") + return pkg_module + elif fullname.startswith(PKG_MODULE_PREFIX_V2) and fullname.endswith(".package"): + return fullname.rsplit(".", 2)[-2] + return fullname + + class _PrependFileLoader(importlib.machinery.SourceFileLoader): def __init__(self, fullname: str, repo: "Repo", package_name: str) -> None: self.repo = repo @@ -485,7 +506,7 @@ def read(self, stream): """Read this index from a provided file object.""" @abc.abstractmethod - def update(self, pkg_fullname): + def update(self, pkgs_fullname: Set[str]): """Update the index in memory with information about a package.""" @abc.abstractmethod @@ -502,8 +523,8 @@ def _create(self) -> spack.tag.TagIndex: def read(self, stream): self.index = spack.tag.TagIndex.from_json(stream) - def update(self, pkg_fullname): - self.index.update_package(pkg_fullname.split(".")[-1], self.repository) + def update(self, pkgs_fullname: Set[str]): + self.index.update_packages({p.split(".")[-1] for p in pkgs_fullname}, self.repository) def write(self, stream): self.index.to_json(stream) @@ -518,15 +539,14 @@ def _create(self) -> "spack.provider_index.ProviderIndex": def read(self, stream): self.index = spack.provider_index.ProviderIndex.from_json(stream, self.repository) - def update(self, pkg_fullname): - name = pkg_fullname.split(".")[-1] - is_virtual = ( + def update(self, pkgs_fullname: Set[str]): + is_virtual = lambda name: ( not self.repository.exists(name) or self.repository.get_pkg_class(name).virtual ) - if is_virtual: - return - self.index.remove_provider(pkg_fullname) - self.index.update(pkg_fullname) + non_virtual_pkgs_fullname = {p for p in pkgs_fullname if not is_virtual(p.split(".")[-1])} + non_virtual_pkgs_names = {p.split(".")[-1] for p in non_virtual_pkgs_fullname} + self.index.remove_providers(non_virtual_pkgs_names) + self.index.update_packages(non_virtual_pkgs_fullname) def write(self, stream): self.index.to_json(stream) @@ -551,8 +571,8 @@ def read(self, stream): def write(self, stream): self.index.to_json(stream) - def update(self, pkg_fullname): - self.index.update_package(pkg_fullname) + def update(self, pkgs_fullname: Set[str]): + self.index.update_packages(pkgs_fullname) class RepoIndex: @@ -571,12 +591,14 @@ class RepoIndex: def __init__( self, - package_checker: FastPackageChecker, + packages_path: str, + package_checker: "Callable[[], FastPackageChecker]", namespace: str, cache: spack.util.file_cache.FileCache, ): - self.checker = package_checker - self.packages_path = self.checker.packages_path + self._get_checker = package_checker + self._checker: Optional[FastPackageChecker] = None + self.packages_path = packages_path if sys.platform == "win32": self.packages_path = spack.llnl.path.convert_to_posix_path(self.packages_path) self.namespace = namespace @@ -585,6 +607,15 @@ def __init__( self.indexes: Dict[str, Any] = {} self.cache = cache + #: Whether the indexes are up to date with the package repository. + self.is_fresh = False + + @property + def checker(self) -> FastPackageChecker: + if self._checker is None: + self._checker = self._get_checker() + return self._checker + def add_indexer(self, name: str, indexer: Indexer): """Add an indexer to the repo index. @@ -594,17 +625,22 @@ def add_indexer(self, name: str, indexer: Indexer): self.indexers[name] = indexer def __getitem__(self, name): - """Get the index with the specified name, reindexing if needed.""" + """Get an up-to-date index with the specified name.""" + return self.get_index(name, allow_stale=False) + + def get_index(self, name, allow_stale: bool = False): + """Get the index with the specified name. The index will be updated if it is stale, unless + allow_stale is True, in which case its contents are not validated against the package + repository. When no cache is available, the index will be updated regardless of the value + of allow_stale.""" indexer = self.indexers.get(name) if not indexer: raise KeyError("no such index: %s" % name) - - if name not in self.indexes: - self._build_all_indexes() - + if name not in self.indexes or (not allow_stale and not self.is_fresh): + self._build_all_indexes(allow_stale=allow_stale) return self.indexes[name] - def _build_all_indexes(self): + def _build_all_indexes(self, allow_stale: bool = False) -> None: """Build all the indexes at once. We regenerate *all* indexes whenever *any* index needs an update, @@ -612,44 +648,50 @@ def _build_all_indexes(self): can take tens of seconds to regenerate sequentially, and we'd rather only pay that cost once rather than on several invocations.""" + is_fresh = True for name, indexer in self.indexers.items(): - self.indexes[name] = self._build_index(name, indexer) + is_fresh &= self._update_index(name, indexer, allow_stale=allow_stale) + self.is_fresh = is_fresh - def _build_index(self, name: str, indexer: Indexer): - """Determine which packages need an update, and update indexes.""" + def _update_index(self, name: str, indexer: Indexer, allow_stale: bool = False) -> bool: + """Determine which packages need an update, and update indexes. Returns true if the + index is fresh.""" # Filename of the provider index cache (we assume they're all json) from spack.spec import SPECFILE_FORMAT_VERSION cache_filename = f"{name}/{self.namespace}-specfile_v{SPECFILE_FORMAT_VERSION}-index.json" - # Compute which packages needs to be updated in the cache - index_mtime = self.cache.mtime(cache_filename) - needs_update = self.checker.modified_since(index_mtime) + with self.cache.read_transaction(cache_filename) as f: + # Get the mtime of the cache if it exists, of -inf. + index_mtime = os.fstat(f.fileno()).st_mtime if f is not None else -math.inf - index_existed = self.cache.init_entry(cache_filename) - if index_existed and not needs_update: - # If the index exists and doesn't need an update, read it - with self.cache.read_transaction(cache_filename) as f: + if f is not None and allow_stale: + # Cache exists and caller accepts stale data: skip the expensive modified_since. indexer.read(f) + self.indexes[name] = indexer.index + return False - else: - # Otherwise update it and rewrite the cache file - with self.cache.write_transaction(cache_filename) as (old, new): - indexer.read(old) if old else indexer.create() + needs_update = self.checker.modified_since(index_mtime) - # Compute which packages needs to be updated **again** in case someone updated them - # while we waited for the lock - new_index_mtime = self.cache.mtime(cache_filename) - if new_index_mtime != index_mtime: - needs_update = self.checker.modified_since(new_index_mtime) - - for pkg_name in needs_update: - indexer.update(f"{self.namespace}.{pkg_name}") + if f is not None and not needs_update: + # Cache exists and is up to date. + indexer.read(f) + self.indexes[name] = indexer.index + return True - indexer.write(new) + # Cache is missing or stale: acquire write lock and rebuild. + with self.cache.write_transaction(cache_filename) as (old, new): + old_mtime = os.fstat(old.fileno()).st_mtime if old is not None else -math.inf + # Re-check in case another writer updated the index while we waited for the lock. + if old_mtime != index_mtime: + needs_update = self.checker.modified_since(old_mtime) + indexer.read(old) if old is not None else indexer.create() + indexer.update({f"{self.namespace}.{pkg_name}" for pkg_name in needs_update}) + indexer.write(new) - return indexer.index + self.indexes[name] = indexer.index + return True class RepoPath: @@ -665,6 +707,7 @@ def __init__(self, *repos: "Repo") -> None: self.by_namespace = nm.NamespaceTrie() self._provider_index: Optional[spack.provider_index.ProviderIndex] = None self._patch_index: Optional[spack.patch.PatchCache] = None + self._index_is_fresh: bool = False self._tag_index: Optional[spack.tag.TagIndex] = None for repo in repos: @@ -757,11 +800,11 @@ def first_repo(self) -> Optional["Repo"]: """Get the first repo in precedence order.""" return self.repos[0] if self.repos else None - @spack.llnl.util.lang.memoized + @memoized def _all_package_names_set(self, include_virtuals) -> Set[str]: return {name for repo in self.repos for name in repo.all_package_names(include_virtuals)} - @spack.llnl.util.lang.memoized + @memoized def _all_package_names(self, include_virtuals: bool) -> List[str]: """Return all unique package names in all repositories.""" return sorted(self._all_package_names_set(include_virtuals), key=lambda n: n.lower()) @@ -811,17 +854,49 @@ def tag_index(self) -> spack.tag.TagIndex: self._tag_index.merge(repo.tag_index) return self._tag_index - @property - def patch_index(self) -> spack.patch.PatchCache: - """Merged PatchIndex from all Repos in the RepoPath.""" - if self._patch_index is None: - from spack.patch import PatchCache + def get_patch_index(self, allow_stale: bool = False) -> spack.patch.PatchCache: + """Return the merged patch index for all repos in this path. - self._patch_index = PatchCache(repository=self) - for repo in reversed(self.repos): - self._patch_index.update(repo.patch_index) + Args: + allow_stale: if True, return a possibly out-of-date index from cache files, + avoiding filesystem calls to check whether the index is up to date. + """ + if self._patch_index is not None and (self._index_is_fresh or allow_stale): + return self._patch_index + + index = spack.patch.PatchCache(repository=self) + for repo in reversed(self.repos): + index.update(repo.get_patch_index(allow_stale=allow_stale)) + self._patch_index = index + self._index_is_fresh = not allow_stale return self._patch_index + def get_patches_for_package( + self, sha256s: List[str], pkg_cls: Type["spack.package_base.PackageBase"] + ) -> List["spack.patch.Patch"]: + """Look up patches by sha256, trying stale cache first to avoid stat calls. + + Args: + sha256s: ordered list of patch sha256 hashes + pkg_cls: package class the patches belong to + + Returns: + List of Patch objects in the same order as sha256s. + + Raises: + spack.error.PatchLookupError: if a sha256 cannot be found even after a full rebuild. + """ + stale_index = self.get_patch_index(allow_stale=True) + try: + return [ + stale_index.patch_for_package(sha256, pkg_cls, validate=True) for sha256 in sha256s + ] + except spack.error.PatchLookupError: + pass + + current_index = self.get_patch_index(allow_stale=False) + return [current_index.patch_for_package(sha256, pkg_cls) for sha256 in sha256s] + def providers_for(self, virtual: Union[str, "spack.spec.Spec"]) -> List["spack.spec.Spec"]: all_packages = self._all_package_names_set(include_virtuals=False) providers = [ @@ -1156,7 +1231,7 @@ def real_name(self, import_name: str) -> Optional[str]: package directory. From Package API v2.0 there is a one-to-one mapping between Spack package names and Python module names, so there is no guessing. - For Packge API v1.x we support the following one-to-many mappings: + For Package API v1.x we support the following one-to-many mappings: * ``num3proxy`` -> ``3proxy`` * ``foo_bar`` -> ``foo_bar``, ``foo-bar`` @@ -1268,7 +1343,9 @@ def dump_provenance(self, spec: "spack.spec.Spec", path: str) -> None: def index(self) -> RepoIndex: """Construct the index for this repo lazily.""" if self._repo_index is None: - self._repo_index = RepoIndex(self._pkg_checker, self.namespace, cache=self._cache) + self._repo_index = RepoIndex( + self.packages_path, lambda: self._pkg_checker, self.namespace, cache=self._cache + ) self._repo_index.add_indexer("providers", ProviderIndexer(self)) self._repo_index.add_indexer("tags", TagIndexer(self)) self._repo_index.add_indexer("patches", PatchIndexer(self)) @@ -1276,18 +1353,18 @@ def index(self) -> RepoIndex: @property def provider_index(self) -> spack.provider_index.ProviderIndex: - """A provider index with names *specific* to this repo.""" + """A fresh provider index with names *specific* to this repo.""" return self.index["providers"] @property def tag_index(self) -> spack.tag.TagIndex: - """Index of tags and which packages they're defined on.""" + """Fresh index of tags and which packages they're defined on.""" return self.index["tags"] - @property - def patch_index(self) -> spack.patch.PatchCache: - """Index of patches and packages they're defined on.""" - return self.index["patches"] + def get_patch_index(self, allow_stale: bool = False) -> spack.patch.PatchCache: + """Index of patches and packages they're defined on. Set allow_stale is True to bypass + cache validation and return a potentially stale index.""" + return self.index.get_index("patches", allow_stale=allow_stale) def providers_for(self, virtual: Union[str, "spack.spec.Spec"]) -> List["spack.spec.Spec"]: providers = self.provider_index.providers_for(virtual) @@ -1424,6 +1501,14 @@ def get_pkg_class(self, pkg_name: str) -> Type["spack.package_base.PackageBase"] if not isinstance(cls, type): tty.die(f"{pkg_name}.{class_name} is not a class") + # Early exit if no overrides to apply or undo + if ( + not self.overrides.get(pkg_name) + and not hasattr(cls, "overridden_attrs") + and not hasattr(cls, "attrs_exclusively_from_config") + ): + return cls + def defining_class(myclass, name): return next((c for c in myclass.__mro__ if name in c.__dict__), None) @@ -1486,7 +1571,7 @@ def unmarshal(root, cache, overrides): def marshal(self): cache = self._cache - if isinstance(cache, spack.llnl.util.lang.Singleton): + if isinstance(cache, Singleton): cache = cache.instance return self.root, cache, self.overrides @@ -1854,7 +1939,7 @@ def construct( class BrokenRepoDescriptor(RepoDescriptor): """A descriptor for a broken repository, used to indicate errors in the configuration that - aren't fatal untill the repository is used.""" + aren't fatal until the repository is used.""" def __init__(self, name: Optional[str], error: str) -> None: super().__init__(name) @@ -2016,9 +2101,7 @@ def create_and_enable(config: spack.config.Configuration) -> RepoPath: #: Global package repository instance. -PATH: RepoPath = spack.llnl.util.lang.Singleton( - lambda: create_and_enable(spack.config.CONFIG) -) # type: ignore[assignment] +PATH = cast(RepoPath, Singleton(lambda: create_and_enable(spack.config.CONFIG))) # Add the finder to sys.meta_path @@ -2130,9 +2213,9 @@ def __init__( repo = PATH.ensure_unwrapped() # We need to compare the base package name - pkg_name = name.rsplit(".", 1)[-1] + pkg_name = name_from_fullname(name) similar = [] - if isinstance(repo, RepoPath): + if isinstance(repo, (Repo, RepoPath)): try: similar = get_close_matches(pkg_name, repo.all_package_names()) except Exception: diff --git a/lib/spack/spack/repo_migrate.py b/lib/spack/spack/repo_migrate.py index aac1c3212e4312..e777e6910684a6 100644 --- a/lib/spack/spack/repo_migrate.py +++ b/lib/spack/spack/repo_migrate.py @@ -447,7 +447,6 @@ def migrate_v2_imports( elif isinstance(node, ast.ImportFrom): # Keep track of old style spack.pkg imports, to be replaced. if node.module and node.module.startswith("spack.pkg.") and node.level == 0: - depth = node.module.count(".") # not all python versions have end_lineno for ImportFrom diff --git a/lib/spack/spack/report.py b/lib/spack/spack/report.py index a36ea58c8ec654..157e7326322528 100644 --- a/lib/spack/spack/report.py +++ b/lib/spack/spack/report.py @@ -2,11 +2,13 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tools to produce reports of spec installations or tests""" + import collections import gzip import os import time import traceback +from typing import Optional import spack.error @@ -55,7 +57,7 @@ def __init__(self, spec): self.time = None self.timestamp = time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime()) self.properties = [ - Property("architecture", spec.architecture), + Property("architecture", spec.architecture) # Property("compiler", spec.compiler), ] self.packages = [] @@ -98,7 +100,11 @@ def skip(self, msg): self.elapsed_time = 0.0 self.message = msg - def fail(self, exc): + def fetch_log(self, log_path: Optional[str] = None) -> str: + """Fetch the log for this spec record. Subclasses should override.""" + return "" + + def fail(self, exc, log_path: Optional[str] = None): """Record failure based on exception type Errors wrapped by spack.error.InstallError are "failures" @@ -112,14 +118,14 @@ def fail(self, exc): self.result = "error" self.message = str(exc) or "Unknown error" self.exception = traceback.format_exc() - self.stdout = self.fetch_log() + self.message + self.stdout = self.fetch_log(log_path) + self.message assert self._start_time, "Start time is None" self.elapsed_time = time.time() - self._start_time - def succeed(self): + def succeed(self, log_path: Optional[str] = None): """Record success for this spec""" self.result = "success" - self.stdout = self.fetch_log() + self.stdout = self.fetch_log(log_path) assert self._start_time, "Start time is None" self.elapsed_time = time.time() - self._start_time @@ -131,23 +137,63 @@ def __init__(self, spec): super().__init__(spec) self.installed_from_binary_cache = None - def fetch_log(self): - """Install log comes from install prefix on success, or stage dir on failure.""" + def fetch_log(self, log_path: Optional[str] = None) -> str: + """Install log comes from log_path if provided, install prefix, or stage dir.""" try: - if os.path.exists(self._package.install_log_path): - stream = gzip.open(self._package.install_log_path, "rt", encoding="utf-8") + if log_path and os.path.exists(log_path): + stream = open(log_path, encoding="utf-8", errors="replace") + elif os.path.exists(self._package.install_log_path): + stream = gzip.open( + self._package.install_log_path, "rt", encoding="utf-8", errors="replace" + ) else: - stream = open(self._package.log_path, encoding="utf-8") + stream = open(self._package.log_path, encoding="utf-8", errors="replace") with stream as f: return f.read() except OSError: return f"Cannot open log for {self._spec.cshort_spec}" - def succeed(self): - super().succeed() + def succeed(self, log_path: Optional[str] = None): + super().succeed(log_path) self.installed_from_binary_cache = self._package.installed_from_binary_cache +class NullInstallRecord(InstallRecord): + """No-op drop-in for InstallRecord when no reporter is configured. + + Avoids reading log files from disk on every completed build.""" + + def start(self) -> None: + pass + + def succeed(self, log_path: Optional[str] = None) -> None: + pass + + def fail(self, exc, log_path: Optional[str] = None) -> None: + pass + + def skip(self, msg: str = "") -> None: + pass + + +class NullRequestRecord(RequestRecord): + """No-op drop-in for RequestRecord when no reporter is configured. + + Avoids traversing the DAG and accumulating data that will not be reported.""" + + def __init__(self) -> None: + dict.__init__(self) + + def skip_installed(self) -> None: + pass + + def append_record(self, record) -> None: + pass + + def summarize(self) -> None: + pass + + class TestRecord(SpecRecord): """Record class with specialization for test logs.""" @@ -155,11 +201,11 @@ def __init__(self, spec, directory): super().__init__(spec) self.directory = directory - def fetch_log(self): + def fetch_log(self, log_path: Optional[str] = None) -> str: """Get output from test log""" log_file = os.path.join(self.directory, self._package.test_suite.test_log_name(self._spec)) try: - with open(log_file, "r", encoding="utf-8") as stream: + with open(log_file, "r", encoding="utf-8", errors="replace") as stream: return "".join(stream.readlines()) except Exception: return f"Cannot open log for {self._spec.cshort_spec}" diff --git a/lib/spack/spack/reporters/cdash.py b/lib/spack/spack/reporters/cdash.py index 358c121866c301..cd3ac31a3724f2 100644 --- a/lib/spack/spack/reporters/cdash.py +++ b/lib/spack/spack/reporters/cdash.py @@ -174,7 +174,7 @@ def build_report_for_package(self, report_dir, package, duration): report_data[cdash_phase]["loglines"].append(xml.sax.saxutils.escape(line)) # something went wrong pre-cdash "configure" phase b/c we have an exception and only - # "update" was encounterd. + # "update" was encountered. # dump the report in the configure line so teams can see what the issue is if len(phases_encountered) == 1 and package.get("exception"): # TODO this mapping is not ideal since these are pre-configure errors @@ -184,7 +184,7 @@ def build_report_for_package(self, report_dir, package, duration): phases_encountered.append(cdash_phase) log_message = ( - "Pre-configure errors occured in Spack's process that terminated the " + "Pre-configure errors occurred in Spack's process that terminated the " "build process prematurely.\nSpack output::\n{0}".format( xml.sax.saxutils.escape(package["exception"]) ) @@ -203,7 +203,7 @@ def build_report_for_package(self, report_dir, package, duration): for phase in phases_encountered: report_data[phase]["endtime"] = self.endtime report_data[phase]["log"] = "\n".join(report_data[phase]["loglines"]) - errors, warnings = parse_log_events(report_data[phase]["loglines"]) + errors, warnings, _ = parse_log_events(report_data[phase]["loglines"]) # Convert errors to warnings if the package reported success. if package["result"] == "success": @@ -229,9 +229,7 @@ def clean_log_event(event): event["post_context"] = xml.sax.saxutils.escape( "\n".join(event["post_context"]) ) - # source_file and source_line_no are either strings or - # the tuple (None,). Distinguish between these two cases. - if event["source_file"][0] is None: + if event["source_file"] is None: event["source_file"] = "" event["source_line_no"] = "" else: @@ -452,13 +450,13 @@ def upload(self, filename): if self.authtoken: request.add_header("Authorization", "Bearer {0}".format(self.authtoken)) try: - response = web_util.urlopen(request, timeout=SPACK_CDASH_TIMEOUT) - if self.current_package_name not in self.buildIds: - resp_value = io.TextIOWrapper(response, encoding="utf-8").read() - match = self.buildid_regexp.search(resp_value) - if match: - buildid = match.group(1) - self.buildIds[self.current_package_name] = buildid + with web_util.urlopen(request, timeout=SPACK_CDASH_TIMEOUT) as response: + if self.current_package_name not in self.buildIds: + resp_value = io.TextIOWrapper(response, encoding="utf-8").read() + match = self.buildid_regexp.search(resp_value) + if match: + buildid = match.group(1) + self.buildIds[self.current_package_name] = buildid except Exception as e: print(f"Upload to CDash failed: {e}") diff --git a/lib/spack/spack/rewiring.py b/lib/spack/spack/rewiring.py index de7231254ad339..e31e4c04c8b534 100644 --- a/lib/spack/spack/rewiring.py +++ b/lib/spack/spack/rewiring.py @@ -59,7 +59,5 @@ def __init__(self, spliced_spec, build_spec, dep): super().__init__( """Rewire of {0} failed due to missing install of build spec {1} - for spec {2}""".format( - spliced_spec, build_spec, dep - ) + for spec {2}""".format(spliced_spec, build_spec, dep) ) diff --git a/lib/spack/spack/sandbox.py b/lib/spack/spack/sandbox.py new file mode 100644 index 00000000000000..2fda7533ae6891 --- /dev/null +++ b/lib/spack/spack/sandbox.py @@ -0,0 +1,269 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +""" +This module implements an unprivileged sandbox for build environments. + +It enforces path-based filesystem whitelisting and optional network isolation, +dynamically adapting to the host kernel's supported Landlock ABI version. + +By design, to support standard build system behaviors like `try_compile` tests, +read access implicitly includes execution rights. IOCTLs and IPC mechanisms are +left unrestricted to ensure compatibility with compilers, terminal output, and +build jobservers. +""" + +import ctypes +import enum +import os +import platform +import stat +import warnings +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Dict, Union + +import spack.error + +# Linux landlock syscalls +SYSCALL_LANDLOCK_CREATE_RULESET = 444 +SYSCALL_LANDLOCK_ADD_RULE = 445 +SYSCALL_LANDLOCK_RESTRICT_SELF = 446 + +PR_SET_NO_NEW_PRIVS = 38 +LANDLOCK_CREATE_RULESET_VERSION = 1 << 0 +LANDLOCK_RULE_PATH_BENEATH = 1 +LANDLOCK_ACCESS_NET_BIND_TCP = 1 << 0 +LANDLOCK_ACCESS_NET_CONNECT_TCP = 1 << 1 +LANDLOCK_RESTRICT_SELF_TSYNC = 1 << 3 + + +class FSAccess(enum.IntFlag): + EXECUTE = 1 << 0 + WRITE_FILE = 1 << 1 + READ_FILE = 1 << 2 + READ_DIR = 1 << 3 + REMOVE_DIR = 1 << 4 + REMOVE_FILE = 1 << 5 + MAKE_CHAR = 1 << 6 + MAKE_DIR = 1 << 7 + MAKE_REG = 1 << 8 + MAKE_SOCK = 1 << 9 + MAKE_FIFO = 1 << 10 + MAKE_BLOCK = 1 << 11 + MAKE_SYM = 1 << 12 + REFER = 1 << 13 # ABI v2 + TRUNCATE = 1 << 14 # ABI v3 + + +def _check_syscall(result: int, name: str) -> int: + """Raise OSError if a libc syscall returned a negative value. + + Mirrors what Python's stdlib does for syscall-backed os.* functions. + """ + if result < 0: + err = ctypes.get_errno() + raise OSError(err, f"{name}: {os.strerror(err)}") + return result + + +class RulesetAttr(ctypes.Structure): + _fields_ = [ + ("handled_access_fs", ctypes.c_uint64), + ("handled_access_net", ctypes.c_uint64), + ("scoped", ctypes.c_uint64), + ] + + +class PathBeneathAttr(ctypes.Structure): + _fields_ = [("allowed_access", ctypes.c_uint64), ("parent_fd", ctypes.c_int32)] + + +class Sandbox(ABC): + """Abstract base class for sandbox implementations.""" + + def allow_read(self, path: Union[str, Path]): + p = Path(path).absolute() + resolved = p.resolve() + if resolved.exists(): + self._allow_read(p, resolved) + + def allow_write(self, path: Union[str, Path]): + p = Path(path).absolute() + resolved = p.resolve() + if resolved.exists(): + self._allow_write(p, resolved) + + @abstractmethod + def _allow_read(self, original: Path, resolved: Path): ... + + @abstractmethod + def _allow_write(self, original: Path, resolved: Path): ... + + @abstractmethod + def apply(self, block_network: bool = False): ... + + +def _get_write_flags(abi_version: int) -> int: + flags = ( + FSAccess.MAKE_BLOCK + | FSAccess.MAKE_CHAR + | FSAccess.MAKE_DIR + | FSAccess.MAKE_FIFO + | FSAccess.MAKE_REG + | FSAccess.MAKE_SOCK + | FSAccess.MAKE_SYM + | FSAccess.REMOVE_DIR + | FSAccess.REMOVE_FILE + | FSAccess.WRITE_FILE + ) + if abi_version >= 2: + flags |= FSAccess.REFER + if abi_version >= 3: + flags |= FSAccess.TRUNCATE + return flags + + +class LandlockSandbox(Sandbox): + def __init__(self, libc=None): + self.libc = libc if libc is not None else ctypes.CDLL(None, use_errno=True) + self.abi_version = self._get_abi_version() + self.path_rules: Dict[Path, int] = {} + self.write_flags = _get_write_flags(self.abi_version) + self.read_flags = FSAccess.EXECUTE | FSAccess.READ_FILE | FSAccess.READ_DIR + self.dir_flags = ( + FSAccess.MAKE_BLOCK + | FSAccess.MAKE_CHAR + | FSAccess.MAKE_DIR + | FSAccess.MAKE_FIFO + | FSAccess.MAKE_REG + | FSAccess.MAKE_SOCK + | FSAccess.MAKE_SYM + | FSAccess.READ_DIR + | FSAccess.REFER + | FSAccess.REMOVE_DIR + | FSAccess.REMOVE_FILE + ) + + def _get_abi_version(self) -> int: + res = self.libc.syscall( + ctypes.c_long(SYSCALL_LANDLOCK_CREATE_RULESET), + None, + ctypes.c_size_t(0), + ctypes.c_uint32(LANDLOCK_CREATE_RULESET_VERSION), + ) + return _check_syscall(res, "landlock_create_ruleset(version)") + + def _allow_read(self, original: Path, resolved: Path): + current_flags = self.path_rules.get(resolved, 0) + self.path_rules[resolved] = current_flags | self.read_flags + + def _allow_write(self, original: Path, resolved: Path): + current_flags = self.path_rules.get(resolved, 0) + self.path_rules[resolved] = current_flags | self.write_flags | self.read_flags + + def _syscall_create_ruleset(self, handled_access_fs: int, handled_access_net: int) -> int: + attr = RulesetAttr( + handled_access_fs=handled_access_fs, handled_access_net=handled_access_net + ) + return _check_syscall( + self.libc.syscall( + ctypes.c_long(SYSCALL_LANDLOCK_CREATE_RULESET), + ctypes.byref(attr), + ctypes.c_size_t(ctypes.sizeof(attr)), + ctypes.c_uint32(0), + ), + "landlock_create_ruleset", + ) + + def _syscall_add_rule(self, ruleset_fd: int, allowed_access: int, path_fd: int) -> None: + rule = PathBeneathAttr(allowed_access=allowed_access, parent_fd=path_fd) + _check_syscall( + self.libc.syscall( + ctypes.c_long(SYSCALL_LANDLOCK_ADD_RULE), + ctypes.c_int(ruleset_fd), + ctypes.c_int(LANDLOCK_RULE_PATH_BENEATH), + ctypes.byref(rule), + ctypes.c_uint32(0), + ), + "landlock_add_rule", + ) + + def _syscall_restrict_self(self, ruleset_fd: int, tsync_flag: int) -> None: + _check_syscall( + self.libc.syscall( + ctypes.c_long(SYSCALL_LANDLOCK_RESTRICT_SELF), + ctypes.c_int(ruleset_fd), + ctypes.c_uint32(tsync_flag), + ), + "landlock_restrict_self", + ) + + def _prctl_no_new_privs(self) -> None: + _check_syscall( + self.libc.prctl( + ctypes.c_int(PR_SET_NO_NEW_PRIVS), + ctypes.c_ulong(1), + ctypes.c_ulong(0), + ctypes.c_ulong(0), + ctypes.c_ulong(0), + ), + "prctl(PR_SET_NO_NEW_PRIVS)", + ) + + def apply(self, block_network: bool = False): + # Network access requires ABI v4 + if block_network and self.abi_version < 4: + raise SandboxError( + f"Blocking network access requires Landlock ABI v4+ (kernel 6.7+), " + f"but this kernel only supports ABI v{self.abi_version}." + ) + net_flags = ( + LANDLOCK_ACCESS_NET_CONNECT_TCP | LANDLOCK_ACCESS_NET_BIND_TCP if block_network else 0 + ) + try: + self._apply(net_flags) + except OSError as e: + raise SandboxError(f"Failed to apply build sandbox: {e}") from e + + def _apply(self, net_flags: int) -> None: + ruleset_fd = self._syscall_create_ruleset(self.write_flags | self.read_flags, net_flags) + + try: + for path, flags in self.path_rules.items(): + try: + # use O_PATH to get an fd w/o needing permissions, and O_NOFOLLOW to avoid + # TOCTOU issues after we've called resolve() on the path. + fd = os.open(str(path), os.O_PATH | os.O_CLOEXEC | os.O_NOFOLLOW) + except OSError as e: + warnings.warn(f"Cannot allow sandbox access to {path} due to: {e}") + continue + try: + st = os.fstat(fd) + if not stat.S_ISDIR(st.st_mode): + # Strip directory-specific flags + flags &= ~self.dir_flags + self._syscall_add_rule(ruleset_fd, flags, fd) + finally: + os.close(fd) + + # Lock down the current process with this ruleset + self._prctl_no_new_privs() + tsync_flag = LANDLOCK_RESTRICT_SELF_TSYNC if self.abi_version >= 8 else 0 + self._syscall_restrict_self(ruleset_fd, tsync_flag) + finally: + os.close(ruleset_fd) + + +def get_sandbox() -> Sandbox: + if platform.system() != "Linux": + raise SandboxError("Build sandboxing is only supported on Linux") + try: + return LandlockSandbox() + except OSError as e: + raise SandboxError(f"Landlock sandboxing is unavailable: {e}") from e + + +class SandboxError(spack.error.SpackError): + """Raised when the build sandbox cannot be set up or applied.""" diff --git a/lib/spack/spack/schema/__init__.py b/lib/spack/spack/schema/__init__.py index bda7246f8f7daa..94d4e6e171f78b 100644 --- a/lib/spack/spack/schema/__init__.py +++ b/lib/spack/spack/schema/__init__.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This module contains jsonschema files for all of Spack's YAML formats.""" + import copy import typing import warnings @@ -128,7 +129,7 @@ def merge_yaml(dest, source, prepend=False, append=False): parent instead of merging. ``+:`` will extend the default prepend merge strategy to include string concatenation - ``-:`` will change the merge strategy to append, it also includes string concatentation + ``-:`` will change the merge strategy to append, it also includes string concatenation """ def they_are(t): diff --git a/lib/spack/spack/schema/bootstrap.py b/lib/spack/spack/schema/bootstrap.py index 33d384ed5b2657..11334956d965ec 100644 --- a/lib/spack/spack/schema/bootstrap.py +++ b/lib/spack/spack/schema/bootstrap.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for bootstrap.yaml configuration file.""" + from typing import Any, Dict #: Schema of a single source @@ -23,6 +24,18 @@ "required": ["name", "metadata"], } +#: schema for dev bootstrap configuration +_dev_schema: Dict[str, Any] = { + "type": "object", + "description": "Dev Bootstrap configuration", + "properties": { + "enable_source": { + "type": "boolean", + "description": "Enable bootstrapping dev dependencies from source", + } + }, +} + properties: Dict[str, Any] = { "bootstrap": { "type": "object", @@ -48,6 +61,7 @@ "additionalProperties": {"type": "boolean"}, "description": "Controls which sources are enabled for automatic bootstrapping", }, + "dev": _dev_schema, }, } } diff --git a/lib/spack/spack/schema/buildcache_spec.py b/lib/spack/spack/schema/buildcache_spec.py index c52af939e6cff8..f5af6030bf092a 100644 --- a/lib/spack/spack/schema/buildcache_spec.py +++ b/lib/spack/spack/schema/buildcache_spec.py @@ -7,6 +7,7 @@ .. literalinclude:: _spack_root/lib/spack/spack/schema/buildcache_spec.py :lines: 15- """ + from typing import Any, Dict import spack.schema.spec diff --git a/lib/spack/spack/schema/cdash.py b/lib/spack/spack/schema/cdash.py index 49334c6c4ba94e..f6f5523bc33bde 100644 --- a/lib/spack/spack/schema/cdash.py +++ b/lib/spack/spack/schema/cdash.py @@ -6,6 +6,7 @@ .. literalinclude:: ../spack/schema/cdash.py :lines: 13- """ + from typing import Any, Dict #: Properties for inclusion in other schemas diff --git a/lib/spack/spack/schema/ci.py b/lib/spack/spack/schema/ci.py index 4c694c8927250f..f8034338be3c6a 100644 --- a/lib/spack/spack/schema/ci.py +++ b/lib/spack/spack/schema/ci.py @@ -6,6 +6,7 @@ .. literalinclude:: ../spack/schema/ci.py :lines: 16- """ + from typing import Any, Dict # Schema for script fields @@ -31,10 +32,9 @@ ] } -# Additional attributes are allow -# and will be forwarded directly to the -# CI target YAML for each job. -attributes_schema = { +# Additional attributes are allowed and will be forwarded directly to the CI target YAML for each +# job. +ci_job_attributes = { "type": "object", "additionalProperties": True, "properties": { @@ -50,6 +50,8 @@ }, } +ref_ci_job_attributes = {"$ref": "#/definitions/ci_job_attributes"} + submapping_schema = { "type": "object", "additionalProperties": False, @@ -64,8 +66,8 @@ "required": ["match"], "properties": { "match": {"type": "array", "items": {"type": "string"}}, - "build-job": attributes_schema, - "build-job-remove": attributes_schema, + "build-job": ref_ci_job_attributes, + "build-job-remove": ref_ci_job_attributes, }, }, }, @@ -101,7 +103,10 @@ def job_schema(name: str): return { "type": "object", "additionalProperties": False, - "properties": {f"{name}-job": attributes_schema, f"{name}-job-remove": attributes_schema}, + "properties": { + f"{name}-job": ref_ci_job_attributes, + f"{name}-job-remove": ref_ci_job_attributes, + }, } @@ -142,5 +147,6 @@ def job_schema(name: str): "title": "Spack CI configuration file schema", "type": "object", "additionalProperties": False, + "definitions": {"ci_job_attributes": ci_job_attributes}, "properties": properties, } diff --git a/lib/spack/spack/schema/compilers.py b/lib/spack/spack/schema/compilers.py index 8d91505fce0d4e..2bf16ccbb2e5f5 100644 --- a/lib/spack/spack/schema/compilers.py +++ b/lib/spack/spack/schema/compilers.py @@ -6,6 +6,7 @@ .. literalinclude:: _spack_root/lib/spack/spack/schema/compilers.py :lines: 15- """ + from typing import Any, Dict import spack.schema.environment @@ -92,7 +93,7 @@ ] }, "implicit_rpaths": implicit_rpaths, - "environment": spack.schema.environment.definition, + "environment": spack.schema.environment.ref_env_modifications, "extra_rpaths": extra_rpaths, }, } @@ -109,4 +110,5 @@ "type": "object", "additionalProperties": False, "properties": properties, + "definitions": {"env_modifications": spack.schema.environment.env_modifications}, } diff --git a/lib/spack/spack/schema/concretizer.py b/lib/spack/spack/schema/concretizer.py index 52da5788dacb0d..19bab4fc03b4d0 100644 --- a/lib/spack/spack/schema/concretizer.py +++ b/lib/spack/spack/schema/concretizer.py @@ -6,6 +6,7 @@ .. literalinclude:: _spack_root/lib/spack/spack/schema/concretizer.py :lines: 12- """ + from typing import Any, Dict LIST_OF_SPECS = {"type": "array", "items": {"type": "string"}} diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py index 1e3f8864f84d9c..188cad095ad17b 100644 --- a/lib/spack/spack/schema/config.py +++ b/lib/spack/spack/schema/config.py @@ -6,6 +6,7 @@ .. literalinclude:: _spack_root/lib/spack/spack/schema/config.py :lines: 17- """ + from typing import Any, Dict import spack.schema @@ -71,7 +72,7 @@ "relocation of binaries (true for max length, integer for specific " "length)", }, - **spack.schema.projections.properties, + **spack.schema.projections.ref_properties, }, }, "install_hash_length": { @@ -89,7 +90,8 @@ }, "develop_stage_link": { "type": "string", - "description": "Name for development spec build stage directories", + "description": "Name for development spec build stage directories. Setting to " + "None will disable develop stage links.", }, "test_stage": { "type": "string", @@ -176,7 +178,7 @@ }, "concurrent_packages": { "type": "integer", - "minimum": 1, + "minimum": 0, "description": "The maximum number of concurrent package builds a single Spack " "instance will run", }, @@ -231,6 +233,31 @@ "enum": ["old", "new"], "description": "Which installer to use. The new installer is experimental.", }, + "sandbox": { + "type": "object", + "description": "Restrict filesystem and network access during builds.", + "additionalProperties": False, + "properties": { + "enable": { + "type": "boolean", + "description": "Enable or disable the build sandbox.", + }, + "allow_network": { + "type": "boolean", + "description": "Allow TCP network access during the build phase.", + }, + "allow_read": { + "type": "array", + "items": {"type": "string"}, + "description": "Additional paths with read and execute permissions.", + }, + "allow_write": { + "type": "array", + "items": {"type": "string"}, + "description": "Additional paths with write and execute permissions.", + }, + }, + }, }, } } @@ -243,6 +270,7 @@ "type": "object", "additionalProperties": False, "properties": properties, + "definitions": {"projections": spack.schema.projections.projections}, } diff --git a/lib/spack/spack/schema/container.py b/lib/spack/spack/schema/container.py index f40368c9062aee..d4055e01cb64fa 100644 --- a/lib/spack/spack/schema/container.py +++ b/lib/spack/spack/schema/container.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for the ``container`` subsection of Spack environments.""" + from typing import Any, Dict _stages_from_dockerhub = { diff --git a/lib/spack/spack/schema/cray_manifest.py b/lib/spack/spack/schema/cray_manifest.py index 987576df2fe40a..922221c8f36db7 100644 --- a/lib/spack/spack/schema/cray_manifest.py +++ b/lib/spack/spack/schema/cray_manifest.py @@ -10,6 +10,7 @@ This does not specify a configuration - it is an input format that is consumed and transformed into Spack DB records. """ + from typing import Any, Dict properties: Dict[str, Any] = { diff --git a/lib/spack/spack/schema/database_index.py b/lib/spack/spack/schema/database_index.py index b72cc17862df6f..5a2d886616f18b 100644 --- a/lib/spack/spack/schema/database_index.py +++ b/lib/spack/spack/schema/database_index.py @@ -7,6 +7,7 @@ .. literalinclude:: _spack_root/lib/spack/spack/schema/database_index.py :lines: 17- """ + from typing import Any, Dict import spack.schema.spec diff --git a/lib/spack/spack/schema/definitions.py b/lib/spack/spack/schema/definitions.py index 5944a9640f79c6..fddbe0cd71edaf 100644 --- a/lib/spack/spack/schema/definitions.py +++ b/lib/spack/spack/schema/definitions.py @@ -7,6 +7,7 @@ .. literalinclude:: _spack_root/lib/spack/spack/schema/definitions.py :lines: 16- """ + from typing import Any, Dict from .spec_list import spec_list_schema diff --git a/lib/spack/spack/schema/develop.py b/lib/spack/spack/schema/develop.py index e02d83a12ac5b7..c6a8ce184d82e5 100644 --- a/lib/spack/spack/schema/develop.py +++ b/lib/spack/spack/schema/develop.py @@ -11,8 +11,7 @@ "additionalProperties": { "type": "object", "additionalProperties": False, - "description": "Name of a package to develop, with its spec and optional " - "source path", + "description": "Name of a package to develop, with its spec and optional source path", "required": ["spec"], "properties": { "spec": { diff --git a/lib/spack/spack/schema/env.py b/lib/spack/spack/schema/env.py index 1283862178009b..267d32399a8e7d 100644 --- a/lib/spack/spack/schema/env.py +++ b/lib/spack/spack/schema/env.py @@ -7,15 +7,49 @@ .. literalinclude:: _spack_root/lib/spack/spack/schema/env.py :lines: 19- """ + +import os from typing import Any, Dict import spack.schema.merged -from .spec_list import spec_list_schema +from .spec_list import spec_list_properties, spec_list_schema #: Top level key in a manifest file TOP_LEVEL_KEY = "spack" +# (DEPRECATED) include concrete entries to be merged under the include key +include_concrete = { + "type": "array", + "default": [], + "description": "List of paths to other environments. Includes concrete specs " + "from their spack.lock files without modifying the source environments. Useful " + "for phased deployments where you want to build on existing concrete specs.", + "items": {"type": "string"}, +} + +group_name_and_deps = { + "group": {"type": "string", "description": "Name for this group of specs"}, + "explicit": { + "type": "boolean", + "default": True, + "description": "When false, specs in this group are installed as implicit " + "dependencies and are eligible for garbage collection.", + }, + "needs": { + "type": "array", + "description": "Groups of specs that are needed by this group", + "items": {"type": "string"}, + }, + "override": { + "type": "object", + "description": "Top-most configuration scope for this group of specs", + "additionalProperties": False, + "properties": {**spack.schema.merged.ref_sections}, + }, +} + + properties: Dict[str, Any] = { "spack": { "type": "object", @@ -25,17 +59,41 @@ "additionalProperties": False, "properties": { # merged configuration scope schemas - **spack.schema.merged.properties, + **spack.schema.merged.ref_sections, # extra environment schema properties - "specs": spec_list_schema, - "include_concrete": { + "specs": { "type": "array", + "description": "List of specs to include in the environment, " + "supporting both simple specs and matrix configurations", "default": [], - "description": "List of paths to other environments. Includes concrete specs " - "from their spack.lock files without modifying the source environments. Useful " - "for phased deployments where you want to build on existing concrete specs.", - "items": {"type": "string"}, + "items": { + "anyOf": [ + { + "type": "object", + "description": "Matrix configuration for generating multiple specs" + " from combinations of constraints", + "additionalProperties": False, + "properties": {**spec_list_properties}, + }, + {"type": "string", "description": "Simple spec string"}, + {"type": "null"}, + { + "type": "object", + "description": "User spec group with a single matrix", + "additionalProperties": False, + "properties": {**spec_list_properties, **group_name_and_deps}, + }, + { + "type": "object", + "description": "User spec group with multiple matrices", + "additionalProperties": False, + "properties": {**group_name_and_deps, "specs": spec_list_schema}, + }, + ] + }, }, + # (DEPRECATED) include concrete to be merged under the include key + "include_concrete": include_concrete, }, } } @@ -46,18 +104,39 @@ "type": "object", "additionalProperties": False, "properties": properties, + "definitions": spack.schema.merged.defs, } -def update(data): - """Update the data in place to remove deprecated properties. +def update(data: Dict[str, Any]) -> bool: + """Update the spack.yaml data to the new format. Args: - data (dict): dictionary to be updated + data: dictionary to be updated Returns: - True if data was changed, False otherwise + ``True`` if data was changed, ``False`` otherwise """ - # There are not currently any deprecated attributes in this section - # that have not been removed - return False + if not isinstance(data, dict): + return False + + if "include_concrete" not in data: + return False + + # Move the old 'include_concrete' paths to reside under the 'include', + # ensuring that the lock file name is appended. + includes = [] + for path in data["include_concrete"]: + if os.path.basename(path) != "spack.lock": + path = os.path.join(path, "spack.lock") + includes.append(path) + + # Now add back the includes the environment file already has. + if "include" in data: + for path in data["include"]: + includes.append(path) + + data["include"] = includes + del data["include_concrete"] + + return True diff --git a/lib/spack/spack/schema/env_vars.py b/lib/spack/spack/schema/env_vars.py index 009961cf5a063c..5c319428ebe8ef 100644 --- a/lib/spack/spack/schema/env_vars.py +++ b/lib/spack/spack/schema/env_vars.py @@ -6,11 +6,12 @@ .. literalinclude:: _spack_root/lib/spack/spack/schema/env_vars.py :lines: 15- """ + from typing import Any, Dict import spack.schema.environment -properties: Dict[str, Any] = {"env_vars": spack.schema.environment.definition} +properties: Dict[str, Any] = {"env_vars": spack.schema.environment.ref_env_modifications} #: Full schema with metadata schema = { @@ -19,4 +20,5 @@ "type": "object", "additionalProperties": False, "properties": properties, + "definitions": {"env_modifications": spack.schema.environment.env_modifications}, } diff --git a/lib/spack/spack/schema/environment.py b/lib/spack/spack/schema/environment.py index 0f0de85d5bbfc5..0882e0fe00970a 100644 --- a/lib/spack/spack/schema/environment.py +++ b/lib/spack/spack/schema/environment.py @@ -4,6 +4,7 @@ """Schema for environment modifications. Meant for inclusion in other schemas. """ + import collections.abc from typing import Any, Dict @@ -12,7 +13,7 @@ "additionalProperties": {"anyOf": [{"type": "string"}, {"type": "number"}]}, } -definition: Dict[str, Any] = { +env_modifications: Dict[str, Any] = { "type": "object", "description": "Environment variable modifications to apply at runtime", "default": {}, @@ -45,6 +46,9 @@ }, } +#: $ref pointer for use in merged schema +ref_env_modifications = {"$ref": "#/definitions/env_modifications"} + def parse(config_obj): """Returns an EnvironmentModifications object containing the modifications diff --git a/lib/spack/spack/schema/include.py b/lib/spack/spack/schema/include.py index 5b2c9dfe9958b9..6938dee47df929 100644 --- a/lib/spack/spack/schema/include.py +++ b/lib/spack/spack/schema/include.py @@ -6,6 +6,7 @@ .. literalinclude:: _spack_root/lib/spack/spack/schema/include.py :lines: 12- """ + from typing import Any, Dict #: Properties for inclusion in other schemas @@ -92,6 +93,7 @@ "description": "List of relative paths within the repository where " "configuration files are located", }, + "name": {"type": "string"}, "when": { "type": "string", "description": "Include this config only when the condition (as " diff --git a/lib/spack/spack/schema/merged.py b/lib/spack/spack/schema/merged.py index 11aefa21cb3947..16c6144a554aed 100644 --- a/lib/spack/spack/schema/merged.py +++ b/lib/spack/spack/schema/merged.py @@ -7,6 +7,7 @@ .. literalinclude:: _spack_root/lib/spack/spack/schema/merged.py :lines: 32- """ + from typing import Any, Dict import spack.schema.bootstrap @@ -19,17 +20,19 @@ import spack.schema.definitions import spack.schema.develop import spack.schema.env_vars +import spack.schema.environment import spack.schema.include import spack.schema.mirrors import spack.schema.modules import spack.schema.packages +import spack.schema.projections import spack.schema.repos import spack.schema.toolchains import spack.schema.upstreams import spack.schema.view #: Properties for inclusion in other schemas -properties: Dict[str, Any] = { +sections: Dict[str, Any] = { **spack.schema.bootstrap.properties, **spack.schema.cdash.properties, **spack.schema.compilers.properties, @@ -50,11 +53,28 @@ **spack.schema.view.properties, } +#: Canonical definitions for JSON Schema $ref +defs: Dict[str, Any] = { + # Section schemas, prefixed to avoid collisions with sub-schema definitions + **{f"section_{name}": schema for name, schema in sections.items()}, + # Sub-schema definitions hoisted for $ref resolution in env.py + "ci_job_attributes": spack.schema.ci.ci_job_attributes, + "env_modifications": spack.schema.environment.env_modifications, + "module_file_configuration": spack.schema.modules.module_file_configuration, + "projections": spack.schema.projections.projections, +} + +#: Properties using $ref pointers into $defs +ref_sections: Dict[str, Any] = { + name: {"$ref": f"#/definitions/section_{name}"} for name in sections +} + #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack merged configuration file schema", "type": "object", "additionalProperties": False, - "properties": properties, + "properties": ref_sections, + "definitions": defs, } diff --git a/lib/spack/spack/schema/mirrors.py b/lib/spack/spack/schema/mirrors.py index 839679bc608bdc..50b23a1dba94fa 100644 --- a/lib/spack/spack/schema/mirrors.py +++ b/lib/spack/spack/schema/mirrors.py @@ -7,6 +7,7 @@ .. literalinclude:: _spack_root/lib/spack/spack/schema/mirrors.py :lines: 13- """ + from typing import Any, Dict #: Common properties for connection specification diff --git a/lib/spack/spack/schema/modules.py b/lib/spack/spack/schema/modules.py index 16f85672ae1bd5..0d7b3cda9be5dd 100644 --- a/lib/spack/spack/schema/modules.py +++ b/lib/spack/spack/schema/modules.py @@ -7,6 +7,7 @@ .. literalinclude:: _spack_root/lib/spack/spack/schema/modules.py :lines: 16- """ + from typing import Any, Dict import spack.schema.environment @@ -67,15 +68,13 @@ "additionalKeysAreSpecs": True, "additionalProperties": {"type": "string"}, # key }, - "environment": { - **spack.schema.environment.definition, - "description": "Custom environment variable modifications to apply in this module " - "file", - }, + "environment": spack.schema.environment.ref_env_modifications, }, } -projections_scheme = spack.schema.projections.properties["projections"] +ref_module_file_configuration = {"$ref": "#/definitions/module_file_configuration"} + +projections_scheme = {"$ref": "#/definitions/projections"} common_props = { "verbose": { @@ -125,10 +124,7 @@ "description": "Custom directory structure and naming convention for module files using " "projection format", }, - "all": { - **module_file_configuration, - "description": "Default configuration applied to all module files in this module set", - }, + "all": ref_module_file_configuration, } tcl_configuration = { @@ -138,7 +134,7 @@ "Lmod", "additionalKeysAreSpecs": True, "properties": {**common_props}, - "additionalProperties": module_file_configuration, + "additionalProperties": ref_module_file_configuration, } lmod_configuration = { @@ -172,7 +168,7 @@ "additionalProperties": array_of_strings, }, }, - "additionalProperties": module_file_configuration, + "additionalProperties": ref_module_file_configuration, } module_config_properties = { @@ -259,5 +255,10 @@ "title": "Spack module file configuration file schema", "type": "object", "additionalProperties": False, + "definitions": { + "module_file_configuration": module_file_configuration, + "projections": spack.schema.projections.projections, + "env_modifications": spack.schema.environment.env_modifications, + }, "properties": properties, } diff --git a/lib/spack/spack/schema/packages.py b/lib/spack/spack/schema/packages.py index a583c543f6c044..4f4e28bd715084 100644 --- a/lib/spack/spack/schema/packages.py +++ b/lib/spack/spack/schema/packages.py @@ -6,6 +6,7 @@ .. literalinclude:: _spack_root/lib/spack/spack/schema/packages.py :lines: 14- """ + from typing import Any, Dict import spack.schema.environment @@ -289,7 +290,7 @@ "patternProperties": {r"^\w": {"type": "string"}}, "additionalProperties": False, }, - "environment": spack.schema.environment.definition, + "environment": spack.schema.environment.ref_env_modifications, "extra_rpaths": extra_rpaths, "implicit_rpaths": implicit_rpaths, "flags": flags, @@ -360,6 +361,7 @@ "type": "object", "additionalProperties": False, "properties": properties, + "definitions": {"env_modifications": spack.schema.environment.env_modifications}, } diff --git a/lib/spack/spack/schema/projections.py b/lib/spack/spack/schema/projections.py index 3704ec3bf30b11..d8952d24959fd4 100644 --- a/lib/spack/spack/schema/projections.py +++ b/lib/spack/spack/schema/projections.py @@ -7,38 +7,41 @@ .. literalinclude:: _spack_root/lib/spack/spack/schema/projections.py :lines: 14- """ + from typing import Any, Dict #: Properties for inclusion in other schemas -properties: Dict[str, Any] = { - "projections": { - "type": "object", - "description": "Customize directory structure and naming schemes by mapping specs to " - "format strings.", - "properties": { - "all": { - "type": "string", - "description": "Default projection format string used as fallback for all specs " - "that do not match other entries. Uses spec format syntax like " - '"{name}/{version}/{hash:16}".', - } - }, - "additionalKeysAreSpecs": True, - "additionalProperties": { +projections: Dict[str, Any] = { + "type": "object", + "description": "Customize directory structure and naming schemes by mapping specs to " + "format strings.", + "properties": { + "all": { "type": "string", - "description": "Projection format string for specs matching this key. Uses spec " - "format syntax supporting tokens like {name}, {version}, {compiler.name}, " - "{^dependency.name}, etc.", - }, - } + "description": "Default projection format string used as fallback for all specs " + "that do not match other entries. Uses spec format syntax like " + '"{name}/{version}/{hash:16}".', + } + }, + "additionalKeysAreSpecs": True, + "additionalProperties": { + "type": "string", + "description": "Projection format string for specs matching this key. Uses spec " + "format syntax supporting tokens like {name}, {version}, {compiler.name}, " + "{^dependency.name}, etc.", + }, } +#: $ref pointer for use in merged schema +ref_properties: Dict[str, Any] = {"projections": {"$ref": "#/definitions/projections"}} + #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack view projection configuration file schema", "type": "object", "additionalProperties": False, - "properties": properties, + "properties": ref_properties, + "definitions": {"projections": projections}, } diff --git a/lib/spack/spack/schema/spec.py b/lib/spack/spack/schema/spec.py index 7ae2a14c31173d..82cfdc48bfdc0e 100644 --- a/lib/spack/spack/schema/spec.py +++ b/lib/spack/spack/schema/spec.py @@ -7,6 +7,7 @@ .. literalinclude:: _spack_root/lib/spack/spack/schema/spec.py :lines: 15- """ + from typing import Any, Dict target = { diff --git a/lib/spack/spack/schema/spec_list.py b/lib/spack/spack/schema/spec_list.py index 3b77b9672244b8..179b5597010fd0 100644 --- a/lib/spack/spack/schema/spec_list.py +++ b/lib/spack/spack/schema/spec_list.py @@ -11,6 +11,15 @@ }, } +spec_list_properties = { + "matrix": matrix_schema, + "exclude": { + "type": "array", + "description": "List of specific spec combinations to exclude from the matrix", + "items": {"type": "string"}, + }, +} + spec_list_schema = { "type": "array", "description": "List of specs to include in the environment, supporting both simple specs and " @@ -23,15 +32,7 @@ "description": "Matrix configuration for generating multiple specs from " "combinations of constraints", "additionalProperties": False, - "properties": { - "matrix": matrix_schema, - "exclude": { - "type": "array", - "description": "List of specific spec combinations to exclude from the " - "matrix", - "items": {"type": "string"}, - }, - }, + "properties": {**spec_list_properties}, }, {"type": "string", "description": "Simple spec string"}, {"type": "null"}, diff --git a/lib/spack/spack/schema/toolchains.py b/lib/spack/spack/schema/toolchains.py index 743abf3df021c2..65701bfdf08c38 100644 --- a/lib/spack/spack/schema/toolchains.py +++ b/lib/spack/spack/schema/toolchains.py @@ -7,6 +7,7 @@ .. literalinclude:: _spack_root/lib/spack/spack/schema/toolchains.py :lines: 13- """ + from typing import Any, Dict #: Properties for inclusion in other schemas diff --git a/lib/spack/spack/schema/url_buildcache_manifest.py b/lib/spack/spack/schema/url_buildcache_manifest.py index e3dc4340fcbc00..15c12a9d59c1ea 100644 --- a/lib/spack/spack/schema/url_buildcache_manifest.py +++ b/lib/spack/spack/schema/url_buildcache_manifest.py @@ -7,6 +7,7 @@ .. literalinclude:: _spack_root/lib/spack/spack/schema/url_buildcache_manifest.py :lines: 11- """ + from typing import Any, Dict properties: Dict[str, Any] = { diff --git a/lib/spack/spack/schema/view.py b/lib/spack/spack/schema/view.py index d809a1cc439cb3..14efe293f40251 100644 --- a/lib/spack/spack/schema/view.py +++ b/lib/spack/spack/schema/view.py @@ -7,9 +7,9 @@ .. literalinclude:: _spack_root/lib/spack/spack/schema/view.py :lines: 15- """ + from typing import Any, Dict -import spack.schema import spack.schema.projections #: Properties for inclusion in other schemas @@ -36,7 +36,20 @@ "properties": { "root": { "type": "string", - "description": "Root directory path where the view will be " "created", + "description": "Root directory path where the view will be created", + }, + "group": { + "oneOf": [ + { + "type": "array", + "items": {"type": "string"}, + "description": "Groups of specs to include in the view", + }, + { + "type": "string", + "description": "Groups of specs to include in the view", + }, + ] }, "link": { "enum": ["roots", "all", "run"], @@ -50,6 +63,12 @@ "description": "How files are linked in the view: 'symlink' " "(default), 'hardlink', or 'copy'", }, + "link_dirs": { + "type": "boolean", + "description": "Whether to link directories in the view, or only files" + " (default: true, only applicable when link_type is 'symlink')", + "default": True, + }, "select": { "type": "array", "items": {"type": "string"}, @@ -62,7 +81,7 @@ "description": "List of specs to exclude from the view " "(default: exclude nothing)", }, - **spack.schema.projections.properties, + **spack.schema.projections.ref_properties, }, }, }, @@ -77,4 +96,5 @@ "type": "object", "additionalProperties": False, "properties": properties, + "definitions": {"projections": spack.schema.projections.projections}, } diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py index dc985c7dbebb7e..4b9dd8d54a7c9e 100644 --- a/lib/spack/spack/solver/asp.py +++ b/lib/spack/spack/solver/asp.py @@ -4,6 +4,7 @@ import collections import collections.abc import enum +import functools import gzip import io import itertools @@ -16,7 +17,6 @@ import sys import time import warnings -from contextlib import contextmanager from typing import ( Any, Callable, @@ -43,7 +43,6 @@ import spack.concretize import spack.config import spack.deptypes as dt -import spack.environment as ev import spack.error import spack.llnl.util.lang import spack.llnl.util.tty as tty @@ -66,12 +65,13 @@ from spack import traverse from spack.compilers.libraries import CompilerPropertyDetector from spack.llnl.util.lang import elide_list +from spack.spec import EMPTY_SPEC from spack.util.compression import GZipFileType from .core import ( AspFunction, AspVar, - NodeArgument, + NodeId, SourceContext, clingo, extract_args, @@ -80,15 +80,13 @@ ) from .input_analysis import create_counter, create_graph_analyzer from .requirements import RequirementKind, RequirementOrigin, RequirementParser, RequirementRule -from .reuse import ReusableSpecsSelector, create_external_parser +from .reuse import ReusableSpecsSelector, SpecFiltersFactory, create_external_parser from .runtimes import RuntimePropertyRecorder, all_libcs, external_config_with_implicit_externals from .versions import Provenance GitOrStandardVersion = Union[vn.GitVersion, vn.StandardVersion] -TransformFunction = Callable[[spack.spec.Spec, List[AspFunction]], List[AspFunction]] - -EMPTY_SPEC = spack.spec.Spec() +TransformFunction = Callable[[str, spack.spec.Spec, List[AspFunction]], List[AspFunction]] class OutputConfiguration(NamedTuple): @@ -119,23 +117,6 @@ def default_clingo_control(): return control -@contextmanager -def named_spec( - spec: Optional[spack.spec.Spec], name: Optional[str] -) -> Iterator[Optional[spack.spec.Spec]]: - """Context manager to temporarily set the name of a spec""" - if spec is None or name is None: - yield spec - return - - old_name = spec.name - spec.name = name - try: - yield spec - finally: - spec.name = old_name - - # Below numbers are used to map names of criteria to the order # they appear in the solution. See concretize.lp @@ -221,16 +202,46 @@ def specify(spec): return spack.spec.Spec(spec) +# Caching because the returned function id is used as a cache key +@functools.lru_cache(maxsize=None) def remove_facts(*to_be_removed: str) -> TransformFunction: """Returns a transformation function that removes facts from the input list of facts.""" - def _remove(spec: spack.spec.Spec, facts: List[AspFunction]) -> List[AspFunction]: - return list(filter(lambda x: x.args[0] not in to_be_removed, facts)) + def _remove(name: str, spec: spack.spec.Spec, facts: List[AspFunction]) -> List[AspFunction]: + return [x for x in facts if x.args[0] not in to_be_removed] return _remove -def dag_closure_by_deptype(spec: spack.spec.Spec, facts: List[AspFunction]) -> List[AspFunction]: +def identity_for_facts( + name: str, spec: spack.spec.Spec, facts: List[AspFunction] +) -> List[AspFunction]: + return facts + + +# Caching because the returned function id is used as a cache key +@functools.lru_cache(maxsize=None) +def dependency_holds( + *, dependency_flags: dt.DepFlag, pkg_cls: Type[spack.package_base.PackageBase] +) -> TransformFunction: + def _transform_fn( + name: str, input_spec: spack.spec.Spec, requirements: List[AspFunction] + ) -> List[AspFunction]: + result = remove_facts("node", "virtual_node")(name, input_spec, requirements) + [ + fn.attr("dependency_holds", pkg_cls.name, name, dt.flag_to_string(t)) + for t in dt.ALL_FLAGS + if t & dependency_flags + ] + if name not in pkg_cls.extendees: + return result + return result + [fn.attr("extends", pkg_cls.name, name)] + + return _transform_fn + + +def dag_closure_by_deptype( + name: str, spec: spack.spec.Spec, facts: List[AspFunction] +) -> List[AspFunction]: edges = spec.edges_to_dependencies() # Compute the "link" transitive closure with `when: root ^[deptypes=link] ` if len(edges) == 1: @@ -312,8 +323,7 @@ def check_packages_exist(specs): class Result: """Result of an ASP solve.""" - def __init__(self, specs, asp=None): - self.asp = asp + def __init__(self, specs): self.satisfiable = None self.optimal = None self.warnings = None @@ -367,10 +377,10 @@ def unsolved_specs(self): return self._unsolved_specs @property - def specs_by_input(self): + def specs_by_input(self) -> Dict[spack.spec.Spec, spack.spec.Spec]: if self._concrete_specs_by_input is None: self._compute_specs_from_answer_set() - return self._concrete_specs_by_input + return self._concrete_specs_by_input # type: ignore def _compute_specs_from_answer_set(self): if not self.satisfiable: @@ -428,11 +438,10 @@ def to_dict(self) -> dict: Does not include anything related to unsatisfiability as we are only interested in storing satisfiable results """ - serial_node_arg = ( - lambda node_dict: f"""{{"id": "{node_dict.id}", "pkg": "{node_dict.pkg}"}}""" + serial_node_arg = lambda node_dict: ( + f"""{{"id": "{node_dict.id}", "pkg": "{node_dict.pkg}"}}""" ) ret = dict() - ret["asp"] = self.asp ret["criteria"] = self.criteria ret["optimal"] = self.optimal ret["warnings"] = self.warnings @@ -461,7 +470,7 @@ def from_dict(obj: dict): def _dict_to_node_argument(dict): id = dict["id"] pkg = dict["pkg"] - return NodeArgument(id=id, pkg=pkg) + return NodeId(id=id, pkg=pkg) def _str_to_spec(spec_str): return spack.spec.Spec(spec_str) @@ -472,13 +481,12 @@ def _dict_to_spec(spec_dict): spack.spec.Spec.ensure_no_deprecated(loaded_spec) return loaded_spec - asp = obj.get("asp") spec_list = obj.get("abstract_specs") if not spec_list: raise RuntimeError("Invalid json for concretization Result object") if spec_list: spec_list = [_str_to_spec(x) for x in spec_list] - result = Result(spec_list, asp) + result = Result(spec_list) criteria = obj.get("criteria") result.criteria = ( @@ -507,7 +515,6 @@ def _dict_to_spec(spec_dict): def __eq__(self, other): eq = ( - self.asp == other.asp, self.satisfiable == other.satisfiable, self.optimal == other.optimal, self.warnings == other.warnings, @@ -606,7 +613,7 @@ def _stats_from_cache(self, cache_entry_file: str) -> Union[Dict, None]: """Returns concretization statistic from the concretization associated with the cache. - Deserialzes the the json representation of the + Deserializes the the json representation of the statistics covering the cached concretization run and returns the Python data structures """ @@ -634,13 +641,13 @@ def _safe_remove(self, cache_dir: pathlib.Path) -> bool: except OSError as e: # Catch other timing/access related issues tty.debug( - f"Exception occured while attempting to remove Concretization Cache entry, {e}" + f"Exception occurred while attempting to remove Concretization Cache entry, {e}" ) pass return False def _lock(self, path: pathlib.Path) -> lk.Lock: - """Returns a lock over the byte range correspnding to the hash of the asp problem. + """Returns a lock over the byte range corresponding to the hash of the asp problem. ``path`` is a path to a file in the cache, and its basename is the hash of the problem. @@ -737,10 +744,7 @@ def fetch(self, problem: str) -> Union[Tuple[Result, Dict], Tuple[None, None]]: # update mod/access time for use w/ LRU cleanup os.utime(cache_path) - return ( - self._results_from_cache(cache_content), - self._stats_from_cache(cache_content), - ) # type: ignore + return (self._results_from_cache(cache_content), self._stats_from_cache(cache_content)) # type: ignore def _is_checksummed_git_version(v): @@ -842,7 +846,7 @@ def handle_error(self, msg, *args): msg = msg.format(*msg_args) # For variant formatting, we sometimes have to construct specs - # to format values properly. Find/replace all occurances of + # to format values properly. Find/replace all occurrences of # Spec(...) with the string representation of the spec mentioned specs_to_construct = re.findall(r"Spec\(([^)]*)\)", msg) for spec_str in specs_to_construct: @@ -858,7 +862,7 @@ def message(self, errors) -> str: input_specs = ", ".join(elide_list([f"`{s}`" for s in self.input_specs], 5)) header = f"failed to concretize {input_specs} for the following reasons:" messages = ( - f" {idx+1:2}. {self.handle_error(msg, *args)}" + f" {idx + 1:2}. {self.handle_error(msg, *args)}" for idx, (_, msg, args) in enumerate(errors) ) return "\n".join((header, *messages)) @@ -955,7 +959,7 @@ def _run_clingo( fetch a result from cache. See ``solve()`` for caching and setup logic. """ # We could just take the cache_key and add it to clingo (since it is the - # full problem representation), but we load conrol files separately as it + # full problem representation), but we load control files separately as it # makes clingo give us better, file-aware error messages. with timer.measure("load"): # Add the problem instance @@ -1132,6 +1136,8 @@ def solve( result, concretization_stats = self._conc_cache.fetch(cache_key) timer.stop("cache-check") + tty.debug("Starting concretizer") + # run the solver and store the result, if it wasn't cached already if not result: problem_repr = "\n".join(problem) @@ -1235,7 +1241,7 @@ def __iter__(self): class ConstraintOrigin(enum.Enum): - """Generates identifiers that can be pased into the solver attached + """Generates identifiers that can be passed into the solver attached to constraints, and then later retrieved to determine the origin of those constraints when ``SpecBuilder`` creates Specs from the solve result. @@ -1326,12 +1332,6 @@ def impose_context(self) -> ConditionIdContext: return ctxt -def _track_dependencies( - input_spec: spack.spec.Spec, requirements: List[AspFunction] -) -> List[AspFunction]: - return requirements + [fn.attr("track_dependencies", input_spec.name)] - - class SpackSolverSetup: """Class to set up and run a Spack concretization solve.""" @@ -1359,11 +1359,9 @@ def __init__(self, tests: spack.concretize.TestsType = False): self.rejected_compilers: Set[spack.spec.Spec] = set() self.possible_oses: Set = set() self.variant_values_from_specs: Set = set() - self.version_constraints: Set = set() + self.version_constraints: Dict[str, Set] = collections.defaultdict(set) self.target_constraints: Set = set() self.default_targets: List = [] - self.compiler_version_constraints: Set = set() - self.post_facts: List = [] self.variant_ids_by_def_id: Dict[int, int] = {} self.reusable_and_possible: ConcreteSpecsByHash = ConcreteSpecsByHash() @@ -1405,7 +1403,10 @@ def pkg_version_rules(self, pkg: Type[spack.package_base.PackageBase]) -> None: # Set the deprecation penalty, according to the package. This should be enough to move the # first version last if deprecated. - self.gen.fact(fn.pkg_fact(pkg.name, fn.version_deprecation_penalty(len(ordered_versions)))) + if ordered_versions: + self.gen.fact( + fn.pkg_fact(pkg.name, fn.version_deprecation_penalty(len(ordered_versions))) + ) for weight, declared_version in enumerate(ordered_versions): self.gen.fact(fn.pkg_fact(pkg.name, fn.version_declared(declared_version, weight))) @@ -1424,11 +1425,12 @@ def pkg_version_rules(self, pkg: Type[spack.package_base.PackageBase]) -> None: for v in sorted(deprecated): self.gen.fact(fn.pkg_fact(pkg.name, fn.deprecated_version(v))) - def spec_versions(self, spec: spack.spec.Spec) -> List[AspFunction]: + def spec_versions( + self, spec: spack.spec.Spec, *, name: Optional[str] = None + ) -> List[AspFunction]: """Return list of clauses expressing spec's version constraints.""" - name = spec.name - msg = "Internal Error: spec with no name occured. Please report to the spack maintainers." - assert name, msg + name = spec.name or name + assert name, "Internal Error: spec with no name occurred. Please file an issue." if spec.concrete: return [fn.attr("version", name, spec.version)] @@ -1437,11 +1439,14 @@ def spec_versions(self, spec: spack.spec.Spec) -> List[AspFunction]: return [] # record all version constraints for later - self.version_constraints.add((name, spec.versions)) + self.version_constraints[name].add(spec.versions) return [fn.attr("node_version_satisfies", name, spec.versions)] - def target_ranges(self, spec: spack.spec.Spec, single_target_fn) -> List[AspFunction]: - name = spec.name + def target_ranges( + self, spec: spack.spec.Spec, single_target_fn, *, name: Optional[str] = None + ) -> List[AspFunction]: + name = spec.name or name + assert name, "Internal Error: spec with no name occurred. Please file an issue." target = spec.architecture.target # Check if the target is a concrete target @@ -1523,10 +1528,6 @@ def pkg_rules(self, pkg, tests): self.package_requirement_rules(pkg) - # trigger and effect tables - self.trigger_rules() - self.effect_rules() - def trigger_rules(self): """Flushes all the trigger rules collected so far, and clears the cache.""" if not self._trigger_cache: @@ -1600,8 +1601,9 @@ def define_variant( # Deal with variants that use validator functions if variant_def.values_defined_by_validator(): - for value in default_values: + for penalty, value in enumerate(default_values, 1): pkg_fact(fn.variant_possible_value(vid, value)) + pkg_fact(fn.variant_penalty(vid, value, penalty)) self.gen.newline() return @@ -1683,7 +1685,8 @@ def variant_rules(self, pkg: Type[spack.package_base.PackageBase]): def _get_condition_id( self, - named_cond: spack.spec.Spec, + name: str, + cond: spack.spec.Spec, cache: ConditionSpecCache, body: bool, context: ConditionIdContext, @@ -1698,17 +1701,18 @@ def _get_condition_id( The id of the cached trigger or effect. """ - pkg_cache = cache[named_cond.name] + pkg_cache = cache[name] + cond_str = str(cond) if cond.name else f"{name} {cond}" + named_cond_key = (cond_str, context.transform) - named_cond_key = (str(named_cond), context.transform) result = pkg_cache.get(named_cond_key) if result: return result[0] cond_id = next(self._id_counter) - requirements = self.spec_clauses(named_cond, body=body, context=context) + requirements = self.spec_clauses(cond, name=name, body=body, context=context) if context.transform: - requirements = context.transform(named_cond, requirements) + requirements = context.transform(name, cond, requirements) pkg_cache[named_cond_key] = (cond_id, requirements) return cond_id @@ -1732,38 +1736,39 @@ def _condition_clauses( context = ConditionContext() context.transform_imposed = remove_facts("node", "virtual_node") - if imposed_spec: - imposed_name = imposed_spec.name or imposed_name - if not imposed_name: - raise ValueError(f"Must provide a name for imposed constraint: '{imposed_spec}'") - - with named_spec(required_spec, required_name), named_spec(imposed_spec, imposed_name): - # Check if we can emit the requirements before updating the condition ID counter. - # In this way, if a condition can't be emitted but the exception is handled in the - # caller, we won't emit partial facts. - - condition_id = next(self._id_counter) - requirement_context = context.requirement_context() - trigger_id = self._get_condition_id( - required_spec, cache=self._trigger_cache, body=True, context=requirement_context - ) - clauses.append(fn.pkg_fact(required_spec.name, fn.condition(condition_id))) - clauses.append(fn.condition_reason(condition_id, msg)) - clauses.append( - fn.pkg_fact(required_spec.name, fn.condition_trigger(condition_id, trigger_id)) - ) - if not imposed_spec: - return clauses, condition_id + # Check if we can emit the requirements before updating the condition ID counter. + # In this way, if a condition can't be emitted but the exception is handled in the + # caller, we won't emit partial facts. + condition_id = next(self._id_counter) + requirement_context = context.requirement_context() + trigger_id = self._get_condition_id( + required_name, + required_spec, + cache=self._trigger_cache, + body=True, + context=requirement_context, + ) + clauses.append(fn.pkg_fact(required_name, fn.condition(condition_id))) + clauses.append(fn.condition_reason(condition_id, msg)) + clauses.append(fn.pkg_fact(required_name, fn.condition_trigger(condition_id, trigger_id))) + if not imposed_spec: + return clauses, condition_id - impose_context = context.impose_context() - effect_id = self._get_condition_id( - imposed_spec, cache=self._effect_cache, body=False, context=impose_context - ) - clauses.append( - fn.pkg_fact(required_spec.name, fn.condition_effect(condition_id, effect_id)) - ) + imposed_name = imposed_spec.name or imposed_name + if not imposed_name: + raise ValueError(f"Must provide a name for imposed constraint: '{imposed_spec}'") + + impose_context = context.impose_context() + effect_id = self._get_condition_id( + imposed_name, + imposed_spec, + cache=self._effect_cache, + body=False, + context=impose_context, + ) + clauses.append(fn.pkg_fact(required_name, fn.condition_effect(condition_id, effect_id))) - return clauses, condition_id + return clauses, condition_id def condition( self, @@ -1803,21 +1808,21 @@ def condition( return condition_id - def package_provider_rules(self, pkg): + def package_provider_rules(self, pkg: Type[spack.package_base.PackageBase]) -> None: for vpkg_name in pkg.provided_virtual_names(): if vpkg_name not in self.possible_virtuals: continue self.gen.fact(fn.pkg_fact(pkg.name, fn.possible_provider(vpkg_name))) for when, provided in pkg.provided.items(): - for vpkg in sorted(provided): + for vpkg in sorted(provided): # type: ignore[type-var] if vpkg.name not in self.possible_virtuals: continue - msg = f"{pkg.name} provides {vpkg} when {when}" + msg = f"{pkg.name} provides {vpkg}{'' if when == EMPTY_SPEC else f' when {when}'}" condition_id = self.condition(when, vpkg, required_name=pkg.name, msg=msg) self.gen.fact( - fn.pkg_fact(when.name, fn.provider_condition(condition_id, vpkg.name)) + fn.pkg_fact(pkg.name, fn.provider_condition(condition_id, vpkg.name)) ) self.gen.newline() @@ -1834,7 +1839,6 @@ def package_provider_rules(self, pkg): def package_dependencies_rules(self, pkg): """Translate ``depends_on`` directives into ASP logic.""" - for cond, deps_by_name in pkg.dependencies.items(): cond_str = str(cond) cond_str_suffix = f" when {cond_str}" if cond_str else "" @@ -1854,35 +1858,12 @@ def package_dependencies_rules(self, pkg): continue msg = f"{pkg.name} depends on {dep.spec}{cond_str_suffix}" - - def dependency_holds( - input_spec: spack.spec.Spec, requirements: List[AspFunction] - ) -> List[AspFunction]: - # TODO: `dependency_holds` is used as a cache key, and is a unique object in - # every iteration of the loop. This prevents deduplication of identical - # "effects" when unique when specs impose the same dependency. We cannot move - # this out of the loop, because the effect cache is keyed only by a spec, and - # not by the dependency type. - result = remove_facts("node", "virtual_node")(input_spec, requirements) + [ - fn.attr( - "dependency_holds", pkg.name, input_spec.name, dt.flag_to_string(t) - ) - for t in dt.ALL_FLAGS - if t & depflag - ] - if input_spec.name not in pkg.extendees: - return result - return result + [fn.attr("extends", pkg.name, input_spec.name)] - context = ConditionContext() context.source = ConstraintOrigin.append_type_suffix( pkg.name, ConstraintOrigin.DEPENDS_ON ) - context.transform_required = _track_dependencies - context.transform_imposed = dependency_holds - + context.transform_imposed = dependency_holds(dependency_flags=depflag, pkg_cls=pkg) self.condition(cond, dep.spec, required_name=pkg.name, msg=msg, context=context) - self.gen.newline() def _gen_match_variant_splice_constraints( @@ -1917,59 +1898,56 @@ def package_splice_rules(self, pkg): for i, (cond, (spec_to_splice, match_variants)) in enumerate( sorted(pkg.splice_specs.items()) ): - with named_spec(cond, pkg.name): - self.version_constraints.add((cond.name, cond.versions)) - self.version_constraints.add((spec_to_splice.name, spec_to_splice.versions)) - hash_var = AspVar("Hash") - splice_node = fn.node(AspVar("NID"), cond.name) - when_spec_attrs = [ - fn.attr(c.args[0], splice_node, *(c.args[2:])) - for c in self.spec_clauses(cond, body=True, required_from=None) - if c.args[0] != "node" - ] - splice_spec_hash_attrs = [ - fn.hash_attr(hash_var, *(c.args)) - for c in self.spec_clauses(spec_to_splice, body=True, required_from=None) - if c.args[0] != "node" - ] - if match_variants is None: - variant_constraints = [] - elif match_variants == "*": - filt_match_variants = set() - for map in pkg.variants.values(): - for k in map: - filt_match_variants.add(k) - filt_match_variants = sorted(filt_match_variants) - variant_constraints = self._gen_match_variant_splice_constraints( - pkg, cond, spec_to_splice, hash_var, splice_node, filt_match_variants - ) - else: - if any( - v in cond.variants or v in spec_to_splice.variants for v in match_variants - ): - raise spack.error.PackageError( - "Overlap between match_variants and explicitly set variants" - ) - variant_constraints = self._gen_match_variant_splice_constraints( - pkg, cond, spec_to_splice, hash_var, splice_node, match_variants - ) - - rule_head = fn.abi_splice_conditions_hold( - i, splice_node, spec_to_splice.name, hash_var + self.version_constraints[pkg.name].add(cond.versions) + self.version_constraints[spec_to_splice.name].add(spec_to_splice.versions) + hash_var = AspVar("Hash") + splice_node = fn.node(AspVar("NID"), pkg.name) + when_spec_attrs = [ + fn.attr(c.args[0], splice_node, *(c.args[2:])) + for c in self.spec_clauses(cond, name=pkg.name, body=True, required_from=None) + if c.args[0] != "node" + ] + splice_spec_hash_attrs = [ + fn.hash_attr(hash_var, *(c.args)) + for c in self.spec_clauses(spec_to_splice, body=True, required_from=None) + if c.args[0] != "node" + ] + if match_variants is None: + variant_constraints = [] + elif match_variants == "*": + filt_match_variants = set() + for map in pkg.variants.values(): + for k in map: + filt_match_variants.add(k) + filt_match_variants = sorted(filt_match_variants) + variant_constraints = self._gen_match_variant_splice_constraints( + pkg, cond, spec_to_splice, hash_var, splice_node, filt_match_variants ) - rule_body_components = ( - [ - # splice_set_fact, - fn.attr("node", splice_node), - fn.installed_hash(spec_to_splice.name, hash_var), - ] - + when_spec_attrs - + splice_spec_hash_attrs - + variant_constraints + else: + if any(v in cond.variants or v in spec_to_splice.variants for v in match_variants): + raise spack.error.PackageError( + "Overlap between match_variants and explicitly set variants" + ) + variant_constraints = self._gen_match_variant_splice_constraints( + pkg, cond, spec_to_splice, hash_var, splice_node, match_variants ) - rule_body = ",\n ".join(str(r) for r in rule_body_components) - rule = f"{rule_head} :-\n {rule_body}." - self.gen.append(rule) + + rule_head = fn.abi_splice_conditions_hold( + i, splice_node, spec_to_splice.name, hash_var + ) + rule_body_components = ( + [ + # splice_set_fact, + fn.attr("node", splice_node), + fn.installed_hash(spec_to_splice.name, hash_var), + ] + + when_spec_attrs + + splice_spec_hash_attrs + + variant_constraints + ) + rule_body = ",\n ".join(str(r) for r in rule_body_components) + rule = f"{rule_head} :-\n {rule_body}." + self.gen.append(rule) self.gen.newline() @@ -2172,6 +2150,7 @@ def spec_clauses( self, spec: spack.spec.Spec, *, + name: Optional[str] = None, body: bool = False, transitive: bool = True, expand_hashes: bool = False, @@ -2190,6 +2169,7 @@ def spec_clauses( try: clauses = self._spec_clauses( spec, + name=spec.name or name, body=body, transitive=transitive, expand_hashes=expand_hashes, @@ -2208,6 +2188,7 @@ def _spec_clauses( self, spec: spack.spec.Spec, *, + name: Optional[str] = None, body: bool = False, transitive: bool = True, expand_hashes: bool = False, @@ -2220,6 +2201,7 @@ def _spec_clauses( Arguments: spec: the spec to analyze + name: optional fallback of spec.name (used for anonymous roots) body: if True, generate clauses to be used in rule bodies (final values) instead of rule heads (setters). transitive: if False, don't generate clauses from dependencies (default True) @@ -2227,7 +2209,7 @@ def _spec_clauses( concrete_build_deps: if False, do not include pure build deps of concrete specs (as they have no effect on runtime constraints) include_runtimes: generate full dependency clauses from runtime libraries that - are ommitted from the solve. + are omitted from the solve. context: tracks what constraint this clause set is generated for (e.g. a ``depends_on`` constraint in a package.py file) seen: set of ids of specs that have already been processed (for internal use only) @@ -2240,7 +2222,7 @@ def _spec_clauses( """ clauses = [] seen = seen if seen is not None else set() - name = spec.name + name = spec.name or name or "" seen.add(id(spec)) f: Union[Type[_Head], Type[_Body]] = _Body if body else _Head @@ -2252,7 +2234,7 @@ def _spec_clauses( if spec.namespace: clauses.append(f.namespace(name, spec.namespace)) - clauses.extend(self.spec_versions(spec)) + clauses.extend(self.spec_versions(spec, name=name)) # seed architecture at the root (we'll propagate later) # TODO: use better semantics. @@ -2263,7 +2245,7 @@ def _spec_clauses( if arch.os: clauses.append(f.node_os(name, arch.os)) if arch.target: - clauses.extend(self.target_ranges(spec, f.node_target)) + clauses.extend(self.target_ranges(spec, f.node_target, name=name)) # variants for vname, variant in sorted(spec.variants.items()): @@ -2332,15 +2314,18 @@ def _spec_clauses( if spec.external: clauses.append(fn.attr("external", name)) + # TODO: a loop over `edges_to_dependencies` is preferred over `edges_from_dependents` + # since dependents can point to specs out of scope for the solver. edges = spec.edges_from_dependents() - virtuals = sorted( - {x for x in itertools.chain.from_iterable([edge.virtuals for edge in edges])} - ) if not body and not spec.concrete: + virtuals = sorted(set(itertools.chain.from_iterable(edge.virtuals for edge in edges))) for virtual in virtuals: clauses.append(fn.attr("provider_set", name, virtual)) clauses.append(fn.attr("virtual_node", virtual)) else: + # direct dependencies are handled under `edges_to_dependencies()` + virtual_iter = (edge.virtuals for edge in edges if not edge.direct) + virtuals = sorted(set(itertools.chain.from_iterable(virtual_iter))) for virtual in virtuals: clauses.append(fn.attr("virtual_on_incoming_edges", name, virtual)) @@ -2440,6 +2425,9 @@ def _spec_clauses( for dependency_type in dt.flag_to_tuple(dspec.depflag): edge_clauses.append(fn.attr("depends_on", name, dep.name, dependency_type)) + for virtual in dspec.virtuals: + dependency_clauses.append(fn.attr("virtual_on_edge", name, dep.name, virtual)) + # By default, wrap head of rules, unless the context says otherwise wrap_node_requirement = body is False if context and context.wrap_node_requirement is not None: @@ -2488,17 +2476,17 @@ def define_package_versions_and_validate_preferences( from_packages_yaml: List[GitOrStandardVersion] = [] for vstr in packages_yaml[pkg_name]["version"]: - v = vn.ver(vstr) + cfg_ver = vn.ver(vstr) - if isinstance(v, vn.GitVersion): - if not require_checksum or v.is_commit: - from_packages_yaml.append(v) + if isinstance(cfg_ver, vn.GitVersion): + if not require_checksum or cfg_ver.is_commit: + from_packages_yaml.append(cfg_ver) else: - matches = [x for x in self.possible_versions[pkg_name] if x.satisfies(v)] + matches = [x for x in self.possible_versions[pkg_name] if x.satisfies(cfg_ver)] matches.sort(reverse=True) if not matches: raise spack.error.ConfigError( - f"Preference for version {v} does not match any known " + f"Preference for version {cfg_ver} does not match any known " f"version of {pkg_name}" ) from_packages_yaml.extend(matches) @@ -2613,8 +2601,15 @@ def target_defaults(self, specs): if not spec.architecture or not spec.architecture.target: continue - target = spack.vendor.archspec.cpu.TARGETS.get(spec.target.name) + target_name = spec.target.name + target = spack.vendor.archspec.cpu.TARGETS.get(target_name) if not target: + if spec.architecture.target_concrete: + raise spack.error.SpecError( + f"the target '{target_name}' in '{spec} is not a known target. " + f"Run 'spack arch --known-targets' to see valid targets." + ) + # range/list constraint (contains ':' or ','): keep existing path self.target_ranges(spec, None) continue @@ -2674,22 +2669,43 @@ def target_defaults(self, specs): def define_version_constraints(self): """Define what version_satisfies(...) means in ASP logic.""" - for pkg_name, versions in self.possible_versions.items(): - for v in versions: + sorted_versions = {} + for pkg_name in self.possible_versions: + possible_versions = list(self.possible_versions[pkg_name]) + possible_versions.sort() + sorted_versions[pkg_name] = possible_versions + for idx, v in enumerate(possible_versions): + self.gen.fact(fn.pkg_fact(pkg_name, fn.version_order(v, idx))) if v in self.git_commit_versions[pkg_name]: sha = self.git_commit_versions[pkg_name].get(v) if sha: self.gen.fact(fn.pkg_fact(pkg_name, fn.version_has_commit(v, sha))) else: self.gen.fact(fn.pkg_fact(pkg_name, fn.version_needs_commit(v))) + self.gen.newline() self.gen.newline() - for pkg_name, versions in self.version_constraints: - # generate facts for each package constraint and the version - # that satisfies it - for v in self.possible_versions[pkg_name]: - if v.satisfies(versions): - self.gen.fact(fn.pkg_fact(pkg_name, fn.version_satisfies(versions, v))) + for pkg_name, set_of_versions in sorted(self.version_constraints.items()): + possible_versions = sorted_versions.get(pkg_name) + if possible_versions is None: + continue + for versions in sorted(set_of_versions): + # Look for contiguous ranges of versions that satisfy the constraint + start_idx = None + for current_idx, v in enumerate(possible_versions): + if v.satisfies(versions): + if start_idx is None: + start_idx = current_idx + elif start_idx is not None: + # End of a contiguous satisfying range found + version_range = fn.version_range(versions, start_idx, current_idx - 1) + self.gen.fact(fn.pkg_fact(pkg_name, version_range)) + start_idx = None + if start_idx is not None: + version_range = fn.version_range( + versions, start_idx, len(possible_versions) - 1 + ) + self.gen.fact(fn.pkg_fact(pkg_name, version_range)) self.gen.newline() def collect_virtual_constraints(self): @@ -2697,42 +2713,31 @@ def collect_virtual_constraints(self): Must be called before define_version_constraints(). """ - # aggregate constraints into per-virtual sets - constraint_map = collections.defaultdict(lambda: set()) - for pkg_name, versions in self.version_constraints: - if not spack.repo.PATH.is_virtual(pkg_name): - continue - constraint_map[pkg_name].add(versions) # extract all the real versions mentioned in version ranges def versions_for(v): if isinstance(v, vn.StandardVersion): - return [v] + yield v elif isinstance(v, vn.ClosedOpenRange): - return [v.lo, vn._prev_version(v.hi)] + yield v.lo + yield vn._prev_version(v.hi) elif isinstance(v, vn.VersionList): - return sum((versions_for(e) for e in v), []) + for e in v: + yield from versions_for(e) else: raise TypeError(f"expected version type, found: {type(v)}") - # define a set of synthetic possible versions for virtuals, so - # that `version_satisfies(Package, Constraint, Version)` has the - # same semantics for virtuals as for regular packages. - for pkg_name, versions in sorted(constraint_map.items()): - possible_versions = set(sum([versions_for(v) for v in versions], [])) - for version in sorted(possible_versions): - self.possible_versions[pkg_name][version].append(Provenance.VIRTUAL_CONSTRAINT) + # Define a set of synthetic possible versions for virtuals that don't define versions in a + # package.py file. This ensures that `version_satisfies(Package, Constraint, Version)` has + # the same semantics for virtuals as for regular packages. + for pkg_name, versions in self.version_constraints.items(): + # Not a virtual package + if pkg_name not in self.possible_virtuals: + continue - def define_compiler_version_constraints(self): - for constraint in sorted(self.compiler_version_constraints): - for compiler_id, compiler in enumerate(self.possible_compilers): - if compiler.spec.satisfies(constraint): - self.gen.fact( - fn.compiler_version_satisfies( - constraint.name, constraint.versions, compiler_id - ) - ) - self.gen.newline() + possible_versions = {pv for v in versions for pv in versions_for(v)} + for version in possible_versions: + self.possible_versions[pkg_name][version].append(Provenance.VIRTUAL_CONSTRAINT) def define_target_constraints(self): def _all_targets_satisfiying(single_constraint): @@ -2904,6 +2909,9 @@ def setup( Return: A ProblemInstanceBuilder populated with facts and rules for an ASP solve. """ + # TODO: remove this local import and get rid of dependency on globals + import spack.environment as ev + reuse = reuse or [] if packages_with_externals is None: packages_with_externals = external_config_with_implicit_externals(spack.config.CONFIG) @@ -2918,12 +2926,11 @@ def setup( candidate_compilers, self.rejected_compilers = possible_compilers( configuration=spack.config.CONFIG ) - for x in candidate_compilers: - if x.external or x in reuse: - continue - reuse.append(x) - for dep in x.traverse(root=False, deptype="run"): - reuse.extend(dep.traverse(deptype=("link", "run"))) + reuse_from_compilers = traverse.traverse_nodes( + [x for x in candidate_compilers if not x.external], deptype=("link", "run") + ) + reused_set = set(reuse) + reuse += [x for x in reuse_from_compilers if x not in reused_set] candidate_compilers.update(compilers_from_reuse) self.possible_compilers = list(candidate_compilers) @@ -3029,6 +3036,10 @@ def setup( self.pkg_rules(pkg, tests=self.tests) self.preferred_variants(pkg) + self.gen.h1("Condition Triggers and Imposed Effects") + self.trigger_rules() + self.effect_rules() + self.gen.h1("Special variants") self.define_auto_variant("dev_path", multi=False) self.define_auto_variant("commit", multi=False) @@ -3051,16 +3062,16 @@ def setup( self.collect_virtual_constraints() self.define_version_constraints() - self.gen.h1("Compiler Version Constraints") - self.define_compiler_version_constraints() - self.gen.h1("Target Constraints") self.define_target_constraints() # once we've done a full traversal and know possible versions, check that the # requested solve is at least consistent. - self.impossible_dependencies_check(specs) - self.input_spec_version_check(specs, allow_deprecated) + # do not check dependency and version availability for already concrete specs + # as they come from reusable specs + abstract_specs = [s for s in specs if not s.concrete] + self.impossible_dependencies_check(abstract_specs) + self.input_spec_version_check(abstract_specs, allow_deprecated) return self.gen @@ -3203,12 +3214,13 @@ def generate_conditional_dep_conditions(self, spec: spack.spec.Spec, condition_i # because reused specs do not track virtual nodes. # Instead, track whether the parent uses the virtual def virtual_handler( - input_spec: spack.spec.Spec, requirements: List[AspFunction] + name: str, input_spec: spack.spec.Spec, requirements: List[AspFunction] ) -> List[AspFunction]: - ret = remove_facts("virtual_node")(input_spec, requirements) + ret = remove_facts("virtual_node")(name, input_spec, requirements) for edge in input_spec.traverse_edges(root=False, cover="edges"): if spack.repo.PATH.is_virtual(edge.spec.name): - ret.append(fn.attr("uses_virtual", edge.parent.name, edge.spec.name)) + parent_name = name if edge.parent is input_spec else edge.parent.name + ret.append(fn.attr("uses_virtual", parent_name, edge.spec.name)) return ret context = ConditionContext() @@ -3217,7 +3229,7 @@ def virtual_handler( ) # Default is to remove node-like attrs, override here context.transform_required = virtual_handler - context.transform_imposed = lambda x, y: y + context.transform_imposed = identity_for_facts try: subcondition_id = self.condition( @@ -3428,7 +3440,7 @@ def possible_compilers(*, configuration) -> Tuple[Set["spack.spec.Spec"], Set["s return result, rejected -FunctionTupleT = Tuple[str, Tuple[Union[str, NodeArgument], ...]] +FunctionTupleT = Tuple[str, Tuple[Union[str, NodeId], ...]] class SpecBuilder: @@ -3445,7 +3457,6 @@ class SpecBuilder: r"^dependency_holds$", r"^package_hash$", r"^root$", - r"^track_dependencies$", r"^uses_virtual$", r"^variant_default_value_from_cli$", r"^virtual_node$", @@ -3456,23 +3467,23 @@ class SpecBuilder: ) @staticmethod - def make_node(*, pkg: str) -> NodeArgument: + def make_node(*, pkg: str) -> NodeId: """Given a package name, returns the string representation of the "min_dupe_id" node in the ASP encoding. Args: pkg: name of a package """ - return NodeArgument(id="0", pkg=pkg) + return NodeId(id="0", pkg=pkg) def __init__(self, specs, hash_lookup=None): - self._specs: Dict[NodeArgument, spack.spec.Spec] = {} + self._specs: Dict[NodeId, spack.spec.Spec] = {} # Matches parent nodes to splice node self._splices: Dict[spack.spec.Spec, List[spack.solver.splicing.Splice]] = {} self._result = None self._command_line_specs = specs - self._flag_sources: Dict[Tuple[NodeArgument, str], Set[str]] = collections.defaultdict( + self._flag_sources: Dict[Tuple[NodeId, str], Set[str]] = collections.defaultdict( lambda: set() ) @@ -3651,15 +3662,11 @@ def _order_index(flag_group): spec.compiler_flags.update({flag_type: ordered_flags}) - def deprecated(self, node: NodeArgument, version: str) -> None: + def deprecated(self, node: NodeId, version: str) -> None: tty.warn(f'using "{node.pkg}@{version}" which is a deprecated version') def splice_at_hash( - self, - parent_node: NodeArgument, - splice_node: NodeArgument, - child_name: str, - child_hash: str, + self, parent_node: NodeId, splice_node: NodeId, child_name: str, child_hash: str ): parent_spec = self._specs[parent_node] splice_spec = self._specs[splice_node] @@ -3669,6 +3676,8 @@ def splice_at_hash( self._splices.setdefault(parent_spec, []).append(splice) def build_specs(self, function_tuples: List[FunctionTupleT]) -> List[spack.spec.Spec]: + # TODO: remove this local import and get rid of dependency on globals + import spack.environment as ev attr_key = { # hash attributes are handled first, since they imply entire concrete specs @@ -3705,7 +3714,7 @@ def build_specs(self, function_tuples: List[FunctionTupleT]) -> List[spack.spec. # predicates on virtual packages. if name != "error": node = args[0] - assert isinstance(node, NodeArgument), ( + assert isinstance(node, NodeId), ( f"internal solver error: expected a node, but got a {type(args[0])}. " "Please report a bug at https://github.com/spack/spack/issues" ) @@ -3813,7 +3822,7 @@ def execute_explicit_splices(self): if not replacement.concrete: replacement.replace_hash() current_spec = current_spec.splice(replacement, transitive) - new_key = NodeArgument(id=key.id, pkg=current_spec.name) + new_key = NodeId(id=key.id, pkg=current_spec.name) specs[new_key] = current_spec return specs @@ -3887,7 +3896,7 @@ class Solver: and passes the setup method to the driver, as well. """ - def __init__(self): + def __init__(self, *, specs_factory: Optional[SpecFiltersFactory] = None): # Compute possible compilers first, so we see them as externals _ = spack.compilers.config.all_compilers(init_config=True) @@ -3900,6 +3909,7 @@ def __init__(self): self.selector = ReusableSpecsSelector( configuration=spack.config.CONFIG, external_parser=create_external_parser(self.packages_with_externals, completion_mode), + factory=specs_factory, packages_with_externals=self.packages_with_externals, ) @@ -3907,6 +3917,8 @@ def __init__(self): def _check_input_and_extract_concrete_specs( specs: Sequence[spack.spec.Spec], ) -> List[spack.spec.Spec]: + _check_unknown_virtuals_in_input_specs(specs) + reusable: List[spack.spec.Spec] = [] analyzer = create_graph_analyzer() for root in specs: @@ -4055,6 +4067,35 @@ def solve_in_rounds( self._conc_cache.cleanup() +class _SkipConcreteVisitor(traverse.BaseVisitor): + """Visitor that trims edges between two concrete nodes.""" + + def neighbors(self, item): + if item.edge.spec.concrete: + return [] + return super().neighbors(item) + + +def _check_unknown_virtuals_in_input_specs(specs: Sequence[spack.spec.Spec]) -> None: + """Raise if any edge in *specs* requires a virtual that does not exist in the repository.""" + errors = [] + for root in specs: + root_edges = traverse.with_artificial_edges([root]) + visitor = traverse.CoverNodesVisitor(_SkipConcreteVisitor()) + for edge in traverse.traverse_breadth_first_edges_generator(root_edges, visitor): + for virtual in edge.virtuals: + if not spack.repo.PATH.is_virtual(virtual): + errors.append(f"'{virtual}' in '{root}' is not a known virtual package") + if not errors: + return + if len(errors) == 1: + raise spack.error.InvalidVirtualOnEdgeError(errors[0]) + details = "\n".join(f" {idx}. {msg}" for idx, msg in enumerate(errors, 1)) + raise spack.error.InvalidVirtualOnEdgeError( + f"unknown virtuals have been found in input specs:\n{details}" + ) + + class UnsatisfiableSpecError(spack.error.UnsatisfiableSpecError): """There was an issue with the spec that was requested (i.e. a user error).""" @@ -4076,7 +4117,6 @@ def __init__(self, msg): class OutputDoesNotSatisfyInputError(InternalConcretizerError): - def __init__( self, input_to_output: List[Tuple[spack.spec.Spec, Optional[spack.spec.Spec]]] ) -> None: diff --git a/lib/spack/spack/solver/concretize.lp b/lib/spack/spack/solver/concretize.lp index ea8d14072d2d65..50b41178d110a3 100644 --- a/lib/spack/spack/solver/concretize.lp +++ b/lib/spack/spack/solver/concretize.lp @@ -76,30 +76,47 @@ attr("namespace", node(ID, Package), Namespace) % Rules on "unification sets", i.e. on sets of nodes allowing a single configuration of any given package unify(SetID, PackageName) :- unification_set(SetID, node(_, PackageName)). -error(1, "Cannot have multiple nodes for {0} in the same unification set {1}", PackageName, SetID) +error(100000, "Cannot have multiple nodes for {0} in the same unification set {1}", PackageName, SetID) :- 2 { unification_set(SetID, node(_, PackageName)) }, unify(SetID, PackageName). unification_set("root", PackageNode) :- attr("root", PackageNode). +unification_set("root", PackageNode) :- attr("virtual_root", PackageNode). + +% A package with a dependency that *is not* build belongs to the same unification set as the parent unification_set(SetID, ChildNode) :- attr("depends_on", ParentNode, ChildNode, Type), Type != "build", unification_set(SetID, ParentNode). -build_only_dependency(ParentNode, node(X, Child)) :- - attr("depends_on", ParentNode, node(X, Child), "build"), - not attr("depends_on", ParentNode, node(X, Child), "link"), - not attr("depends_on", ParentNode, node(X, Child), "run"). +% A separate unification set can be created if any of the build dependencies can be duplicated +needs_build_unification_set(ParentNode) :- attr("depends_on", ParentNode, _, "build"). + +% If any of the build dependencies can be duplicated, they can go into any ("build", ID) set +unification_set(("build", ID), node(X, Child)) + :- attr("depends_on", ParentNode, node(X, Child), "build"), + build_set_id(ParentNode, ID), + needs_build_unification_set(ParentNode). + +% A virtual that is on an edge of non-build type belongs to the same unification set as the parent of the provider +unification_set(SetID, VirtualNode) :- + virtual_on_edge(ParentNode, ProviderNode, VirtualNode, Type), Type != "build", + unification_set(SetID, ParentNode). -unification_set(("build", node(X, Child)), node(X, Child)) - :- build_only_dependency(ParentNode, node(X, Child)), - multiple_unification_sets(Child), - unification_set(_, ParentNode). +% A virtual that is on a build edge, goes in the build set id of the parent of the provider +unification_set(("build", ID), VirtualNode) :- + virtual_on_edge(ParentNode, ProviderNode, VirtualNode, "build"), + build_set_id(ParentNode, ID), + needs_build_unification_set(ParentNode). -unification_set("generic_build", node(X, Child)) - :- build_only_dependency(ParentNode, node(X, Child)), - not multiple_unification_sets(Child), - unification_set(_, ParentNode). +% Needed for reused dependencies. A reused dependency has its build edges trimmed, so we +% only care about the non-build edges. +unification_set(SetID, node(VirtualID, Virtual)) :- + concrete(ParentNode), + attr("virtual_on_edge", ParentNode, ProviderNode, Virtual), + provider(ProviderNode, node(VirtualID, Virtual)), + unification_set(SetID, ParentNode). + +% Limit the number of unification sets to a reasonable number to avoid combinatorial explosion +#const max_build_unification_sets = 4. +1 { build_set_id(ParentNode, 0..max_build_unification_sets-1) } 1 :- needs_build_unification_set(ParentNode). -unification_set(SetID, VirtualNode) - :- provider(PackageNode, VirtualNode), - unification_set(SetID, PackageNode). % Compute sub-sets of the nodes, if requested. These can be either the nodes connected % to another node by "link" edges, or the nodes connected to another node by "link and @@ -268,8 +285,7 @@ error(100, multiple_values_error, Attribute, Package) % Version semantics %----------------------------------------------------------------------------- -% versions are declared w/priority -- declared with priority implies declared -pkg_fact(Package, version_declared(Version)) :- pkg_fact(Package, version_declared(Version, _)). +version_declared(Package, Version) :- pkg_fact(Package, version_order(Version, _)). % If something is a package, it has only one version and that must be a % declared version. @@ -278,14 +294,20 @@ pkg_fact(Package, version_declared(Version)) :- pkg_fact(Package, version_declar % against to ensure they cannot be inferred when a non-error solution is % possible +version_constraint_satisfied(node(ID,Package), Constraint) :- + attr("version", node(ID,Package), Version), + pkg_fact(Package, version_order(Version, VersionIdx)), + pkg_fact(Package, version_range(Constraint, MinIdx, MaxIdx)), + VersionIdx >= MinIdx, VersionIdx <= MaxIdx. + % Pick a single version among the possible ones -1 { choose_version(node(ID, Package), Version) : pkg_fact(Package, version_declared(Version)) } 1 :- attr("node", node(ID, Package)). +1 { choose_version(node(ID, Package), Version) : version_declared(Package, Version) } 1 :- attr("node", node(ID, Package)). % To choose the "fake" version of virtual packages, we need a separate rule. % Note that a virtual node may or may not have a version, but cannot have more than one. -{ choose_version(node(ID, Package), Version) : pkg_fact(Package, version_satisfies(Constraint, Version)) } 1 - :- attr("node_version_satisfies", node(ID, Package), Constraint), - attr("virtual_node", node(ID, Package)). +{ choose_version(node(ID, Package), Version) : version_declared(Package, Version) } 1 + :- attr("virtual_node", node(ID, Package)), + virtual(Package). #defined compiler_package/1. @@ -331,18 +353,16 @@ version_deprecation_penalty(node(ID, Package), Penalty) % More specific error message if the version cannot satisfy some constraint % Otherwise covered by `no_version_error` and `versions_conflict_error`. -error(10000, "Cannot satisfy '{0}@{1}' 1({2})", Package, Constraint, Version) - :- attr("node_version_satisfies", node(ID, Package), Constraint), - attr("version", node(ID, Package), Version), - not pkg_fact(Package, version_satisfies(Constraint, Version)). - error(10000, "Cannot satisfy '{0}@{1}'", Package, Constraint) :- attr("node_version_satisfies", node(ID, Package), Constraint), - not attr("version", node(ID, Package), _). + not version_constraint_satisfied(node(ID,Package), Constraint). attr("node_version_satisfies", node(ID, Package), Constraint) :- attr("version", node(ID, Package), Version), - pkg_fact(Package, version_satisfies(Constraint, Version)). + pkg_fact(Package, version_order(Version, VersionIdx)), + pkg_fact(Package, version_range(Constraint, MinIdx, MaxIdx)), + VersionIdx >= MinIdx, VersionIdx <= MaxIdx. + % if a version needs a commit or has one it can use the commit variant can_accept_commit(Package, Version) :- pkg_fact(Package, version_needs_commit(Version)). @@ -377,7 +397,6 @@ error(10, "Commit '{0}' must match package.py value '{1}' for '{2}@={3}'", Vsha, % A "condition_set(PackageNode, _)" is the set of nodes on which PackageNode can require / impose conditions condition_set(PackageNode, PackageNode) :- attr("node", PackageNode). -condition_set(PackageNode, PackageNode) :- provider(PackageNode, VirtualNode). condition_set(PackageNode, VirtualNode) :- provider(PackageNode, VirtualNode). condition_set(PackageNode, DependencyNode) :- condition_set(PackageNode, PackageNode), depends_on(PackageNode, DependencyNode). condition_set(ID, VirtualNode) :- condition_set(ID, PackageNode), provider(PackageNode, VirtualNode). @@ -470,6 +489,12 @@ satisfied(trigger(PackageNode), condition_requirement("closure", A1, A2, A3)) :- generic_condition_requirement("closure", A1, A2, A3), condition_nodes(PackageNode, node(X, A1)). +satisfied(trigger(PackageNode), condition_requirement("virtual_on_edge", A1, A2, A3)) :- + attr("virtual_on_edge", node(X, A1), node(Y, A2), A3), + generic_condition_requirement("virtual_on_edge", A1, A2, A3), + condition_nodes(PackageNode, node(X, A1)), + condition_nodes(PackageNode, node(Y, A2)). + %%%% % Conditions verified on pure build deps of reused nodes %%%% @@ -500,7 +525,9 @@ satisfied(trigger(node(Hash, Package)), condition_requirement("node_version_sati reused_provider(node(Hash, Package), node(Hash, Language)), hash_attr(Hash, "version", Package, Version), condition_requirement(ID, "node_version_satisfies", Package, VersionConstraint), - pkg_fact(Package, version_satisfies(VersionConstraint, Version)). + pkg_fact(Package, version_order(Version, VersionIdx)), + pkg_fact(Package, version_range(VersionConstraint, MinIdx, MaxIdx)), + VersionIdx >= MinIdx, VersionIdx <= MaxIdx. satisfied(trigger(node(Hash, Package)), condition_requirement(Name, Package, A1, A2)) :- trigger_real_node(ID, node(Hash, Package)), @@ -561,11 +588,11 @@ imposed_packages(ID, A1) :- imposed_constraint(ID, _, A1, _, _). imposed_packages(ID, A1) :- imposed_constraint(ID, _, A1, _, _, _). imposed_packages(ID, A1) :- imposed_constraint(ID, "depends_on", _, A1, _). -imposed_nodes(node(NodeID, Package), node(X, A1)) - :- condition_set(node(NodeID, Package), node(X, A1)), +imposed_nodes(node(ID, Package), node(X, A1)) + :- condition_set(node(ID, Package), node(X, A1)), % We don't want to add build requirements to imposed nodes, to avoid % unsat problems when we deal with self-dependencies: gcc@14 %gcc@10 - not self_build_requirement(node(NodeID, Package), node(X, A1)). + not self_build_requirement(node(ID, Package), node(X, A1)). self_build_requirement(node(X, Package), node(Y, Package)) :- build_requirement(node(X, Package), node(Y, Package)). @@ -585,7 +612,10 @@ attr(Name, node(X, A1), A2, A3, A4) :- impose(ID, PackageNode), imposed_constrai % Provider set is relevant only for literals, since it's the only place where `^[virtuals=foo] bar` % might appear in the HEAD of a rule -attr("provider_set", node(min_dupe_id, Provider), node(min_dupe_id, Virtual)) +1 { + attr("provider_set", node(0..MaxProvider-1, Provider), node(0..MaxVirtual-1, Virtual)): + max_dupes(Provider, MaxProvider), max_dupes(Virtual, MaxVirtual) +} 1 :- solve_literal(TriggerID), trigger_and_effect(_, TriggerID, EffectID), impose(EffectID, _), @@ -636,17 +666,19 @@ attr("concrete_variant_set", node(X, A1), Variant, Value, ID) :- attr("direct_dependency", ParentNode, node_requirement("node_version_satisfies", BuildDependency, Constraint)), concrete_build_requirement(ParentNode, BuildDependency), attr("concrete_build_dependency", ParentNode, BuildDependency, BuildDependencyHash), - not 1 { pkg_fact(BuildDependency, version_satisfies(Constraint, Version)) : hash_attr(BuildDependencyHash, "version", BuildDependency, Version) } 1. + not hash_attr(BuildDependencyHash, "node_version_satisfies", BuildDependency, Constraint). :- attr("direct_dependency", ParentNode, node_requirement("provider_set", BuildDependency, Virtual)), concrete_build_requirement(ParentNode, BuildDependency), attr("concrete_build_dependency", ParentNode, BuildDependency, BuildDependencyHash), not attr("virtual_on_build_edge", ParentNode, BuildDependency, Virtual). -% Give a penalty if reuse introduces a node compiled with a compiler that is not used otherwise +% Give a penalty if reuse introduces a node compiled with a compiler that is not used otherwise. +% The only exception is if the current node is a compiler itself. compiler_from_reuse(Hash, DependencyPackage) :- attr("concrete_build_dependency", ParentNode, DependencyPackage, Hash), attr("virtual_on_build_edge", ParentNode, DependencyPackage, Virtual), + not node_compiler(_, ParentNode), language(Virtual). compiler_penalty_from_reuse(Hash) :- @@ -810,6 +842,9 @@ attr("uses_virtual", PackageNode, Virtual) :- attr("node_flag", node(ID, Package), node_flag(FlagType, Flag, _, _)), not imposed_constraint(Hash, "node_flag", Package, node_flag(FlagType, Flag, _, _)). +% we cannot have two nodes with the same hash +:- attr("hash", PackageNode1, Hash), attr("hash", PackageNode2, Hash), PackageNode1 < PackageNode2. + #defined condition/1. #defined subcondition/2. #defined condition_requirement/3. @@ -839,9 +874,12 @@ concrete(PackageNode) :- attr("hash", PackageNode, _), attr("node", PackageNode) % Dependencies of any type imply that one package "depends on" another depends_on(PackageNode, DependencyNode) :- attr("depends_on", PackageNode, DependencyNode, _). -% a dependency holds if its condition holds and if it is not concrete. % Dependencies of concrete specs don't need to be resolved -- they arise from the concrete specs themselves. -attr("track_dependencies", Node) :- build(Node). +do_not_impose(EffectID, node(X, Package)) :- + trigger_and_effect(Package, _, TriggerID, EffectID), + trigger_condition_holds(TriggerID, node(X, Package)), + imposed_constraint(EffectID, "dependency_holds", Package, _, _), + concrete(node(X, Package)). % If a dependency holds on a package node, there must be one and only one dependency node satisfying it 1 { attr("depends_on", PackageNode, node(0..Y-1, Dependency), Type) : max_dupes(Dependency, Y) } 1 @@ -863,9 +901,8 @@ edge_needed(ParentNode, node(X, Child)) :- attr("dependency_holds", ParentNode, Child, _). virtual_edge_needed(ParentNode, ChildNode, node(X, Virtual)) :- - depends_on(ParentNode, ChildNode), build(ParentNode), - node_depends_on_virtual(ParentNode, Virtual), + attr("virtual_on_edge", ParentNode, ChildNode, Virtual), provider(ChildNode, node(X, Virtual)). virtual_edge_needed(ParentNode, ChildNode, node(X, Virtual)) :- @@ -943,20 +980,19 @@ error(1, Msg) :- % Virtual dependencies %----------------------------------------------------------------------------- -% Enforces all virtuals to be provided, if multiple of them are provided together -error(100, "Package '{0}' needs to provide both '{1}' and '{2}' together, but provides only '{1}'", Package, Virtual1, Virtual2) -:- % This package provides 2 or more virtuals together - condition_holds(ID, node(X, Package)), - pkg_fact(Package, provided_together(ID, SetID, Virtual1)), - pkg_fact(Package, provided_together(ID, SetID, Virtual2)), - Virtual1 != Virtual2, - % One node depends on those virtuals AND on this package - node_depends_on_virtual(ClientNode, Virtual1), - node_depends_on_virtual(ClientNode, Virtual2), - depends_on(ClientNode, node(X, Package)), - % But this package is a provider of only one of them - provider(node(X, Package), node(_, Virtual1)), - not provider(node(X, Package), node(_, Virtual2)). +% Package provides to this client at least one virtual from those that need to be provided together +node_uses_provider_with_constraints(ClientNode, node(X, Package), ID, SetID) :- + condition_holds(ID, node(X, Package)), + pkg_fact(Package, provided_together(ID, SetID, V)), + attr("virtual_on_edge", ClientNode, node(X, Package), V). + +% This error is triggered if the package provides some but not all required virtuals from +% the set that needs to be provided together +error(100, "Package '{0}' needs to also provide '{1}' (provided_together constraint)", Package, Virtual) +:- node_uses_provider_with_constraints(ClientNode, node(X, Package), ID, SetID), + pkg_fact(Package, provided_together(ID, SetID, Virtual)), + node_depends_on_virtual(ClientNode, Virtual), + not attr("virtual_on_edge", ClientNode, node(X, Package), Virtual). % if a package depends on a virtual, it's not external and we have a % provider for that virtual then it depends on the provider @@ -965,14 +1001,18 @@ node_depends_on_virtual(PackageNode, Virtual, Type) virtual(Virtual). node_depends_on_virtual(PackageNode, Virtual) :- node_depends_on_virtual(PackageNode, Virtual, Type). +virtual_is_needed(Virtual) :- node_depends_on_virtual(PackageNode, Virtual). -1 { attr("depends_on", PackageNode, ProviderNode, Type) : provider(ProviderNode, node(VirtualID, Virtual)) } 1 +1 { virtual_on_edge(PackageNode, ProviderNode, node(VirtualID, Virtual), Type) : provider(ProviderNode, node(VirtualID, Virtual)) } 1 :- node_depends_on_virtual(PackageNode, Virtual, Type). -attr("virtual_on_edge", PackageNode, ProviderNode, Virtual) - :- attr("dependency_holds", PackageNode, Virtual, Type), - attr("depends_on", PackageNode, ProviderNode, Type), - provider(ProviderNode, node(_, Virtual)). +% A package that depends on a virtual with type ("build", "link") cannot have two providers +% (one for the "build" and one for the "link") +:- node_depends_on_virtual(PackageNode, Virtual), M = #count { VirtualID : virtual_on_edge(PackageNode, ProviderNode, node(VirtualID, Virtual), _) }, M > 1. + + +attr("virtual_on_edge", PackageNode, ProviderNode, Virtual) :- virtual_on_edge(PackageNode, ProviderNode, node(_, Virtual), _). +attr("depends_on", PackageNode, ProviderNode, Type) :- virtual_on_edge(PackageNode, ProviderNode, _, Type). % If a virtual node is in the answer set, it must be either a virtual root, % or used somewhere @@ -990,9 +1030,8 @@ attr("virtual_on_incoming_edges", ProviderNode, Virtual) attr("root", ProviderNode), provider(ProviderNode, node(min_dupe_id, Virtual)). -% dependencies on virtuals also imply that the virtual is a virtual node -1 { attr("virtual_node", node(0..X-1, Virtual)) : max_dupes(Virtual, X) } - :- node_depends_on_virtual(PackageNode, Virtual). +% If a virtual is needed on an edge, at least one virtual node must exist +:- virtual_is_needed(Virtual), not 1 { attr("virtual_node", node(0..X-1, Virtual)) : max_dupes(Virtual, X) }. % If there's a virtual node, we must select one and only one provider. % The provider must be selected among the possible providers. @@ -1287,7 +1326,7 @@ error(50000, Message) :- % Variant semantics %----------------------------------------------------------------------------- % Packages define potentially several definitions for each variant, and depending -% on their attibutes, duplicate nodes for the same package may use different +% on their attributes, duplicate nodes for the same package may use different % definitions. So the variant logic has several jobs: % A. Associate a variant definition with a node, by VariantID % B. Associate defaults and attributes (sticky, etc.) for the selected variant ID with the node. @@ -1297,29 +1336,23 @@ error(50000, Message) :- % Variant definitions come from package facts in two ways: % 1. unconditional variants are always defined on all nodes for a given package -variant_definition(node(NodeID, Package), Name, VariantID) :- +variant_definition(node(ID, Package), Name, VariantID) :- pkg_fact(Package, variant_definition(Name, VariantID)), - attr("node", node(NodeID, Package)). + attr("node", node(ID, Package)). % 2. conditional variants are only defined if the conditions hold for the node -variant_definition(node(NodeID, Package), Name, VariantID) :- +variant_definition(node(ID, Package), Name, VariantID) :- pkg_fact(Package, variant_condition(Name, VariantID, ConditionID)), - condition_holds(ConditionID, node(NodeID, Package)). + condition_holds(ConditionID, node(ID, Package)). % If there are any definitions for a variant on a node, the variant is "defined". variant_defined(PackageNode, Name) :- variant_definition(PackageNode, Name, _). -% We must select one definition for each defined variant on a node. -1 { - node_has_variant(PackageNode, Name, VariantID) : variant_definition(PackageNode, Name, VariantID) -} 1 :- - variant_defined(PackageNode, Name). - % Solver must pick the variant definition with the highest id. When conditions hold % for two or more variant definitions, this prefers the last one defined. -:- node_has_variant(node(NodeID, Package), Name, SelectedVariantID), - variant_definition(node(NodeID, Package), Name, VariantID), - VariantID > SelectedVariantID. +node_has_variant(PackageNode, Name, SelectedVariantID) :- + SelectedVariantID = #max { VariantID : variant_definition(PackageNode, Name, VariantID) }, + variant_defined(PackageNode, Name). % B: Associating applicable package rules with nodes @@ -1333,73 +1366,81 @@ variant_defined(PackageNode, Name) :- variant_definition(PackageNode, Name, _). % packages.yaml and CLI are associated with just the variant name. % Also, settings specified on the CLI apply to all duplicates, but always have % `min_dupe_id` as their node id. -variant_default_value(node(NodeID, Package), VariantName, Value) :- - node_has_variant(node(NodeID, Package), VariantName, VariantID), +variant_default_value(node(ID, Package), VariantName, Value) :- + node_has_variant(node(ID, Package), VariantName, VariantID), pkg_fact(Package, variant_default_value_from_package_py(VariantID, Value)), not variant_default_value_from_packages_yaml(Package, VariantName, _), not attr("variant_default_value_from_cli", node(min_dupe_id, Package), VariantName, _). -variant_default_value(node(NodeID, Package), VariantName, Value) :- - node_has_variant(node(NodeID, Package), VariantName, _), +variant_default_value(node(ID, Package), VariantName, Value) :- + node_has_variant(node(ID, Package), VariantName, _), variant_default_value_from_packages_yaml(Package, VariantName, Value), not attr("variant_default_value_from_cli", node(min_dupe_id, Package), VariantName, _). -variant_default_value(node(NodeID, Package), VariantName, Value) :- - node_has_variant(node(NodeID, Package), VariantName, _), +variant_default_value(node(ID, Package), VariantName, Value) :- + node_has_variant(node(ID, Package), VariantName, _), attr("variant_default_value_from_cli", node(min_dupe_id, Package), VariantName, Value). +% Penalty from the variant definition +possible_variant_penalty(VariantID, Value, Penalty) :- pkg_fact(Package, variant_penalty(VariantID, Value, Penalty)). + +% Use a very high penalty for variant values that are not defined in the package, +% for instance those defined implicitly by a validator. +possible_variant_penalty(VariantID, Value, 100) :- + pkg_fact(Package, variant_possible_value(VariantID, Value)), + not pkg_fact(Package, variant_penalty(VariantID, Value, _)). + variant_penalty(node(NodeID, Package), Variant, Value, Penalty) :- node_has_variant(node(NodeID, Package), Variant, VariantID), attr("variant_value", node(NodeID, Package), Variant, Value), - pkg_fact(Package, variant_penalty(VariantID, Value, Penalty)), + possible_variant_penalty(VariantID, Value, Penalty), not variant_default_value(node(NodeID, Package), Variant, Value), % variants set explicitly from a directive don't count as non-default - not attr("variant_set", node(NodeID, Package), Variant, _), + not attr("variant_set", node(NodeID, Package), Variant, Value), % variant values forced by propagation don't count as non-default - not propagate(node(NodeID, Package), variant_value(Variant, _, _)). + not propagate(node(NodeID, Package), variant_value(Variant, Value, _)). % -- Associate the definition's possible values with the node -variant_possible_value(node(NodeID, Package), VariantName, Value) :- - node_has_variant(node(NodeID, Package), VariantName, VariantID), +variant_possible_value(node(ID, Package), VariantName, Value) :- + node_has_variant(node(ID, Package), VariantName, VariantID), pkg_fact(Package, variant_possible_value(VariantID, Value)). -variant_possible_value(node(NodeID, Package), VariantName, Value) :- - node_has_variant(node(NodeID, Package), VariantName, VariantID), +variant_possible_value(node(ID, Package), VariantName, Value) :- + node_has_variant(node(ID, Package), VariantName, VariantID), pkg_fact(Package, variant_penalty(VariantID, Value, _)). -variant_value_from_disjoint_sets(node(NodeID, Package), VariantName, Value1, Set1) :- - node_has_variant(node(NodeID, Package), VariantName, VariantID), +variant_value_from_disjoint_sets(node(ID, Package), VariantName, Value1, Set1) :- + node_has_variant(node(ID, Package), VariantName, VariantID), pkg_fact(Package, variant_value_from_disjoint_sets(VariantID, Value1, Set1)). % -- Associate definition's arity with the node -variant_single_value(node(NodeID, Package), VariantName) :- - node_has_variant(node(NodeID, Package), VariantName, VariantID), +variant_single_value(node(ID, Package), VariantName) :- + node_has_variant(node(ID, Package), VariantName, VariantID), variant_type(VariantID, VariantType), VariantType != "multi". % C: Determining variant values on each node % if a variant is sticky, but not set, its value is the default value -attr("variant_selected", node(ID, Package), Variant, Value, VariantType, VariantID) :- +attr("variant_value", node(ID, Package), Variant, Value) :- node_has_variant(node(ID, Package), Variant, VariantID), variant_default_value(node(ID, Package), Variant, Value), pkg_fact(Package, variant_sticky(VariantID)), - variant_type(VariantID, VariantType), not attr("variant_set", node(ID, Package), Variant), build(node(ID, Package)). % we can choose variant values from all the possible values for the node 1 { - attr("variant_selected", PackageNode, Variant, Value, VariantType, VariantID) - : variant_possible_value(PackageNode, Variant, Value) + attr("variant_value", PackageNode, Variant, Value) : variant_possible_value(PackageNode, Variant, Value) } :- - node_has_variant(PackageNode, Variant, VariantID), - variant_type(VariantID, VariantType), + node_has_variant(PackageNode, Variant, _), build(PackageNode). -% variant_selected is only needed for reconstruction on the python side, so we can ignore it here -attr("variant_value", PackageNode, Variant, Value) :- - attr("variant_selected", PackageNode, Variant, Value, VariantType, VariantID). +% variant_selected is only needed for reconstruction on the python side +attr("variant_selected", PackageNode, Variant, Value, VariantType, VariantID) :- + attr("variant_value", PackageNode, Variant, Value), + node_has_variant(PackageNode, Variant, VariantID), + variant_type(VariantID, VariantType). % a variant cannot be set if it is not a variant on the package error(100, "Cannot set variant '{0}' for package '{1}' because the variant condition cannot be satisfied for the given spec", Variant, Package) @@ -1469,15 +1510,26 @@ error(100, "{0} variant '{1}' cannot have values '{2}' and '{3}' as they come fr :- attr("variant_set", node(ID, Package), Variant, Value), not attr("variant_value", node(ID, Package), Variant, Value). +% In a case with `variant("foo", when="+bar")` and a user request for +foo or ~foo, +% force +bar to be set too. This gives no penalty if `+bar` is not the default value, and +% optimizes a long chain of deductions that may cause clingo to hang. +attr("variant_set", node(ID, Package), AnotherVariant, AnotherValue) + :- attr("variant_set", node(ID, Package), Variant, Value), + attr("variant_selected", node(ID, Package), Variant, _, _, VariantID), + pkg_fact(Package, variant_condition(Variant, VariantID, ConditionID)), + pkg_fact(Package, condition_trigger(ConditionID, TriggerID)), + condition_requirement(TriggerID,"variant_value",Package, AnotherVariant, AnotherValue), + build(node(ID, Package)). + % A default variant value that is not used, makes sense only for multi valued variants variant_default_not_used(node(ID, Package), Variant, Value) :- variant_default_value(node(ID, Package), Variant, Value), node_has_variant(node(ID, Package), Variant, VariantID), variant_type(VariantID, VariantType), VariantType == "multi", not attr("variant_value", node(ID, Package), Variant, Value), - not propagate(node(ID, Package), variant_value(Variant, _, _)), + not propagate(node(ID, Package), variant_value(Variant, Value, _)), % variant set explicitly don't count for this metric - not attr("variant_set", node(ID, Package), Variant, _), + not attr("variant_set", node(ID, Package), Variant, Value), attr("node", node(ID, Package)). % Treat 'none' in a special way - it cannot be combined with other @@ -1549,10 +1601,9 @@ propagate(ChildNode, PropagatedAttribute, edge_types(DepType1, DepType2)) :- %---- % If a variant is propagated, and can be accepted, set its value -attr("variant_selected", PackageNode, Variant, Value, VariantType, VariantID) :- +attr("variant_value", PackageNode, Variant, Value) :- propagate(PackageNode, variant_value(Variant, Value, _)), - node_has_variant(PackageNode, Variant, VariantID), - variant_type(VariantID, VariantType), + node_has_variant(PackageNode, Variant, _), variant_possible_value(PackageNode, Variant, Value). % If a variant is propagated, we cannot have extraneous values @@ -1562,7 +1613,7 @@ variant_is_propagated(PackageNode, Variant) :- not attr("variant_set", PackageNode, Variant). :- variant_is_propagated(PackageNode, Variant), - attr("variant_selected", PackageNode, Variant, Value, _, _), + attr("variant_value", PackageNode, Variant, Value), not propagate(PackageNode, variant_value(Variant, Value, _)). error(100, "{0} and {1} cannot both propagate variant '{2}' to the shared dependency: {3}", @@ -1707,8 +1758,9 @@ unification_set_compiler("root", node(CompilerHash, Compiler), Language) :- #defined allow_mixing/1. % You can't have >1 compiler for a given language if mixing is disabled -error(100, "Compiler mixing is disabled") :- - #count { CompilerNode : unification_set_compiler("root", CompilerNode, Language) } > 1. +error(100, "Compiler mixing is disabled for the {0} language", Language) :- + language(Language), + #count { CompilerNode : unification_set_compiler("root", CompilerNode, Language) } > 1. %----------------------------------------------------------------------------- % Runtimes @@ -1813,8 +1865,10 @@ compiler(Compiler) :- target_supported(Compiler, _, _). % Can't use targets on node if the compiler for the node doesn't support them language("c"). +language("cuda-lang"). language("cxx"). language("fortran"). +language("hip-lang"). language_runtime("fortran-rt"). error(10, "Only external, or concrete, compilers are allowed for the {0} language", Language) @@ -1898,7 +1952,9 @@ error(100, "Cannot set multiple {0} values for {1} from cli", FlagType, Package) % hash_attrs are versions, but can_splice_attr are usually node_version_satisfies hash_attr(Hash, "node_version_satisfies", PackageName, Constraint) :- hash_attr(Hash, "version", PackageName, Version), - pkg_fact(PackageName, version_satisfies(Constraint, Version)). + pkg_fact(PackageName, version_order(Version, VersionIdx)), + pkg_fact(PackageName, version_range(Constraint, MinIdx, MaxIdx)), + VersionIdx >= MinIdx, VersionIdx <= MaxIdx. % This recovers the exact semantics for hash reuse hash and depends_on are where % splices are decided, and virtual_on_edge can result in name-changes, which is @@ -1925,6 +1981,31 @@ avoid_link_dependency(Hash, DepName) :- compiler_package(PackageName), not compiler_used_as_a_library(node(_, PackageName), Hash). +% When a compiler is not used as a library, its transitive run-type dependencies are unified in the +% build environment (they appear in PATH etc.), but their pure link-type dependencies are NOT. This +% is different from unification sets that include all link/run deps: the goal is to avoid imposing +% transitive link constraints from the toolchain, which would add to max_dupes and require us to +% increase the max_dupes threshold for many packages. + +% Base case: direct run dep of a non-library compiler +compiler_non_lib_run_dep(DepHash, DepName) :- + compiler_package(CompilerName), + not compiler_used_as_a_library(node(_, CompilerName), CompilerHash), + hash_attr(CompilerHash, "depends_on", CompilerName, DepName, "run"), + hash_attr(CompilerHash, "hash", DepName, DepHash). + +% Recursive case: run deps of run deps (full transitive closure) +compiler_non_lib_run_dep(TransDepHash, TransDepName) :- + compiler_non_lib_run_dep(DepHash, DepName), + hash_attr(DepHash, "depends_on", DepName, TransDepName, "run"), + hash_attr(DepHash, "hash", TransDepName, TransDepHash). + +% Pure link deps (link but not run) of any node in that closure are avoided +avoid_link_dependency(DepHash, LinkDepName) :- + compiler_non_lib_run_dep(DepHash, DepName), + hash_attr(DepHash, "depends_on", DepName, LinkDepName, "link"), + not hash_attr(DepHash, "depends_on", DepName, LinkDepName, "run"). + % Without splicing, we simply recover the exact semantics imposed_constraint(ParentHash, "hash", ChildName, ChildHash) :- hash_attr(ParentHash, "hash", ChildName, ChildHash), @@ -1960,7 +2041,7 @@ build(PackageNode) :- attr("node", PackageNode), not concrete(PackageNode). % Minimizing builds is tricky. We want a minimizing criterion -% because we want to reuse what is avaialble, but +% because we want to reuse what is available, but % we also want things that are built to stick to *default preferences* from % the package and from the user. We therefore treat built specs differently and apply % a different set of optimization criteria to them. Spack's *first* priority is to @@ -2022,16 +2103,22 @@ opt_criterion(310, "requirement weight"). }. % Try hard to reuse installed packages (i.e., minimize the number built) -opt_criterion(110, "number of packages to build (vs. reuse)"). -#minimize { 0@110: #true }. -#minimize { 1@110,PackageNode : build(PackageNode), not treat_node_as_concrete(PackageNode) }. +opt_criterion(120, "number of packages to build (vs. reuse)"). +#minimize { 0@120: #true }. +#minimize { 1@120,PackageNode : build(PackageNode), not treat_node_as_concrete(PackageNode) }. -opt_criterion(100, "number of nodes from the same package"). -#minimize { 0@100: #true }. -#minimize { ID@100,Package : attr("node", node(ID, Package)), not self_build_requirement(_, node(ID, Package)) }. -#minimize { ID@100,Package : attr("virtual_node", node(ID, Package)) }. +opt_criterion(110, "number of nodes from the same package"). +#minimize { 0@110: #true }. +#minimize { ID@110,Package : attr("node", node(ID, Package)), not self_build_requirement(_, node(ID, Package)) }. +#minimize { ID@110,Package : attr("virtual_node", node(ID, Package)) }. #defined optimize_for_reuse/0. +% Minimize the unification set ID used for build dependencies. This reduces the number of optimal +% solutions that differ only by which node belongs to which unification set. +opt_criterion(100, "build unification sets"). +#minimize{ 0@100: #true }. +#minimize{ ID@100,ParentNode : build_set_id(ParentNode, ID) }. + % Minimize the number of deprecated versions being used opt_criterion(73, "deprecated versions used"). #minimize{ 0@273: #true }. @@ -2073,16 +2160,6 @@ opt_criterion(65, "variant penalty (roots)"). build_priority(PackageNode, Priority) }. -opt_criterion(60, "preferred providers for roots"). -#minimize{ 0@260: #true }. -#minimize{ 0@60: #true }. -#minimize{ - Weight@60+Priority,ProviderNode,X,Virtual - : provider_weight(ProviderNode, node(X, Virtual), Weight), - attr("root", ProviderNode), not language(Virtual), not language_runtime(Virtual), - build_priority(ProviderNode, Priority) -}. - opt_criterion(55, "default values of variants not being used (roots)"). #minimize{ 0@255: #true }. #minimize{ 0@55: #true }. @@ -2093,59 +2170,59 @@ opt_criterion(55, "default values of variants not being used (roots)"). build_priority(PackageNode, Priority) }. +% Choose the preferred compiler before penalizing variants, to avoid that a variant penalty +% on e.g. gcc causes clingo to pick another compiler e.g. llvm +opt_criterion(48, "preferred compilers"). +#minimize{ 0@248: #true }. +#minimize{ 0@48: #true }. +#minimize{ + Weight@48+Priority,ProviderNode,X,Virtual + : provider_weight(ProviderNode, node(X, Virtual), Weight), + language(Virtual), + build_priority(ProviderNode, Priority) +}. + +opt_criterion(41, "compiler penalty from reuse"). +#minimize{ 0@241: #true }. +#minimize{ 0@41: #true }. +#minimize{1@41,Hash : compiler_penalty_from_reuse(Hash)}. + % Try to use default variants or variants that have been set -opt_criterion(50, "variant penalty (non-roots)"). -#minimize{ 0@250: #true }. -#minimize{ 0@50: #true }. +opt_criterion(40, "variant penalty (non-roots)"). +#minimize{ 0@240: #true }. +#minimize{ 0@40: #true }. #minimize { - Penalty@50+Priority,PackageNode,Variant,Value + Penalty@40+Priority,PackageNode,Variant,Value : variant_penalty(PackageNode, Variant, Value, Penalty), not attr("root", PackageNode), build_priority(PackageNode, Priority) }. -% Minimize the weights of the providers, i.e. use as much as -% possible the most preferred providers -opt_criterion(48, "preferred providers (non-roots)"). -#minimize{ 0@248: #true }. -#minimize{ 0@48: #true }. +% Minimize the weights of all the other providers (mpi, lapack, etc.) +opt_criterion(38, "preferred providers (excluded compilers and language runtimes)"). +#minimize{ 0@238: #true }. +#minimize{ 0@38: #true }. #minimize{ - Weight@48+Priority,ProviderNode,X,Virtual + Weight@38+Priority,ProviderNode,X,Virtual : provider_weight(ProviderNode, node(X, Virtual), Weight), - not attr("root", ProviderNode), not language(Virtual), not language_runtime(Virtual), + not language(Virtual), not language_runtime(Virtual), build_priority(ProviderNode, Priority) }. % Minimize the number of compilers used on nodes - compiler_penalty(PackageNode, C-1) :- C = #count { CompilerNode : node_compiler(PackageNode, CompilerNode) }, node_compiler(PackageNode, _), C > 0. -opt_criterion(46, "number of compilers used on the same node"). -#minimize{ 0@246: #true }. -#minimize{ 0@46: #true }. +opt_criterion(36, "number of compilers used on the same node"). +#minimize{ 0@236: #true }. +#minimize{ 0@36: #true }. #minimize{ - Penalty@46+Priority,PackageNode + Penalty@36+Priority,PackageNode : compiler_penalty(PackageNode, Penalty), build_priority(PackageNode, Priority) }. -opt_criterion(40, "preferred compilers"). -#minimize{ 0@240: #true }. -#minimize{ 0@40: #true }. -#minimize{ - Weight@40+Priority,ProviderNode,X,Virtual - : provider_weight(ProviderNode, node(X, Virtual), Weight), - language(Virtual), - build_priority(ProviderNode, Priority) -}. - -opt_criterion(41, "compiler penalty from reuse"). -#minimize{ 0@241: #true }. -#minimize{ 0@41: #true }. -#minimize{1@41,Hash : compiler_penalty_from_reuse(Hash)}. - opt_criterion(30, "non-preferred OS's"). #minimize{ 0@230: #true }. #minimize{ 0@30: #true }. diff --git a/lib/spack/spack/solver/core.py b/lib/spack/spack/solver/core.py index f31ad7c37c887c..5c98702725e8d4 100644 --- a/lib/spack/spack/solver/core.py +++ b/lib/spack/spack/solver/core.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Low-level wrappers around clingo API and other basic functionality related to ASP""" + import importlib import pathlib from types import ModuleType @@ -213,7 +214,7 @@ def parse_term(*args, **kwargs): return clingo().parse_term(*args, **kwargs) -class NodeArgument(NamedTuple): +class NodeId(NamedTuple): """Represents a node in the DAG""" id: str @@ -230,10 +231,10 @@ class NodeFlag(NamedTuple): def intermediate_repr(sym): """Returns an intermediate representation of clingo models for Spack's spec builder. - Currently, transforms symbols from clingo models either to strings or to NodeArgument objects. + Currently, transforms symbols from clingo models either to strings or to NodeId objects. Returns: - This will turn a ``clingo.Symbol`` into a string or NodeArgument, or a sequence of + This will turn a ``clingo.Symbol`` into a string or NodeId, or a sequence of ``clingo.Symbol`` objects into a tuple of those objects. """ # TODO: simplify this when we no longer have to support older clingo versions. @@ -242,7 +243,7 @@ def intermediate_repr(sym): try: if sym.name == "node": - return NodeArgument( + return NodeId( id=intermediate_repr(sym.arguments[0]), pkg=intermediate_repr(sym.arguments[1]) ) elif sym.name == "node_flag": diff --git a/lib/spack/spack/solver/error_messages.lp b/lib/spack/spack/solver/error_messages.lp index 0e83f3e293ce84..a79fd643ca804a 100644 --- a/lib/spack/spack/solver/error_messages.lp +++ b/lib/spack/spack/solver/error_messages.lp @@ -88,7 +88,7 @@ condition_cause(Condition2, ID2, Condition1, ID1) :- % More specific error message if the version cannot satisfy some constraint % Otherwise covered by `no_version_error` and `versions_conflict_error`. -error(10000, "Cannot satisfy '{0}@{1}' 3({2})", Package, Constraint, Version, startcauses, ConstraintCause, CauseID) +error(10000, "Cannot satisfy '{0}@{1}' (selected version {2} does not match)", Package, Constraint, Version, startcauses, ConstraintCause, CauseID) :- attr("node_version_satisfies", node(ID, Package), Constraint), pkg_fact(TriggerPkg, condition_effect(ConstraintCause, EffectID)), imposed_constraint(EffectID, "node_version_satisfies", Package, Constraint), @@ -97,7 +97,7 @@ error(10000, "Cannot satisfy '{0}@{1}' 3({2})", Package, Constraint, Version, st not pkg_fact(Package, version_satisfies(Constraint, Version)), choose_version(node(ID, Package), Version). -error(100, "Cannot satisfy '{0}@{1}' 4({2})", Package, Constraint, Version, startcauses, ConstraintCause, CauseID) +error(100, "Cannot satisfy '{0}@{1}' (version {2} does not match)", Package, Constraint, Version, startcauses, ConstraintCause, CauseID) :- attr("node_version_satisfies", node(ID, Package), Constraint), pkg_fact(TriggerPkg, condition_effect(ConstraintCause, EffectID)), imposed_constraint(EffectID, "node_version_satisfies", Package, Constraint), @@ -158,6 +158,11 @@ error(0, Msg, startcauses, TriggerID, ID1, ConstraintID, ID2) unification_set(X, node(ID1, TriggerPackage)), build(node(ID, Package)). % ignore conflicts for concrete packages +pkg_fact(Package, version_satisfies(Constraint, Version)) :- + pkg_fact(Package, version_order(Version, VersionIdx)), + pkg_fact(Package, version_range(Constraint, MinIdx, MaxIdx)), + VersionIdx >= MinIdx, VersionIdx <= MaxIdx. + % variables to show #show error/2. #show error/3. diff --git a/lib/spack/spack/solver/heuristic.lp b/lib/spack/spack/solver/heuristic.lp index 1baed82b7091f6..15dde1f936c48a 100644 --- a/lib/spack/spack/solver/heuristic.lp +++ b/lib/spack/spack/solver/heuristic.lp @@ -13,6 +13,7 @@ #heuristic attr("version", node(PackageID, Package), Version). [80, level] #heuristic attr("variant_value", PackageNode, Variant, Value). [80, level] #heuristic attr("node_target", node(PackageID, Package), Target). [80, level] +#heuristic virtual_on_edge(PackageNode, ProviderNode, Virtual, Type). [80, level] #heuristic attr("virtual_node", node(X, Virtual)). [600, init] #heuristic attr("virtual_node", node(X, Virtual)). [-1, sign] diff --git a/lib/spack/spack/solver/input_analysis.py b/lib/spack/spack/solver/input_analysis.py index 647bb5eb820d67..77075ae65a6841 100644 --- a/lib/spack/spack/solver/input_analysis.py +++ b/lib/spack/spack/solver/input_analysis.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Classes to analyze the input of a solve, and provide information to set up the ASP problem""" + import collections from typing import Dict, List, NamedTuple, Set, Tuple, Union @@ -17,6 +18,7 @@ import spack.store from spack.error import SpackError from spack.llnl.util import lang, tty +from spack.spec import EMPTY_SPEC class PossibleGraph(NamedTuple): @@ -85,10 +87,9 @@ def is_virtual(self, name: str) -> bool: def is_allowed_on_this_platform(self, *, pkg_name: str) -> bool: """Returns true if a package is allowed on the current host""" pkg_cls = self.repo.get_pkg_class(pkg_name) - no_condition = spack.spec.Spec() for when_spec, conditions in pkg_cls.requirements.items(): # Restrict analysis to unconditional requirements - if when_spec != no_condition: + if when_spec != EMPTY_SPEC: continue for requirements, _, _ in conditions: if not any(x.intersects(self._platform_condition) for x in requirements): @@ -165,44 +166,55 @@ def possible_dependencies( continue pkg_cls = self.repo.get_pkg_class(pkg_name=pkg_name) - for name, conditions in pkg_cls.dependencies_by_name(when=True).items(): - if all(self.unreachable(pkg_name=pkg_name, when_spec=x) for x in conditions): - tty.debug( - f"[{__name__}] Not adding {name} as a dep of {pkg_name}, because " - f"conditions cannot be met" - ) - continue + for when_spec, dependencies in pkg_cls.dependencies.items(): + # Check if we need to process this condition at all. We can skip the unreachable + # check if all dependencies in this condition are already accounted for. + new_dependencies: List[str] = [] + for name, dep in dependencies.items(): + if strict_depflag: + if dep.depflag != allowed_deps: + continue + elif not (dep.depflag & allowed_deps): + continue - if not self._has_deptypes( - conditions, allowed_deps=allowed_deps, strict=strict_depflag - ): - continue + if name in edges[pkg_name] or name in virtuals: + continue - if name in virtuals: + new_dependencies.append(name) + + if not new_dependencies: continue - dep_names = set() - if self.is_virtual(name): - virtuals.add(name) - if expand_virtuals: - providers = self.providers_for(name) - dep_names = {spec.name for spec in providers} - else: - dep_names = {name} + if self.unreachable(pkg_name=pkg_name, when_spec=when_spec): + tty.debug( + f"[{__name__}] Skipping {', '.join(new_dependencies)} dependencies of " + f"{pkg_name}, because {when_spec} is not met" + ) + continue - edges[pkg_name].update(dep_names) + for name in new_dependencies: + dep_names: Set[str] = set() + if self.is_virtual(name): + virtuals.add(name) + if expand_virtuals: + providers = self.providers_for(name) + dep_names = {spec.name for spec in providers} + else: + dep_names = {name} - if not transitive: - continue + edges[pkg_name].update(dep_names) - for dep_name in dep_names: - if dep_name in edges: + if not transitive: continue - if not self._is_possible(pkg_name=dep_name): - continue + for dep_name in dep_names: + if dep_name in edges: + continue + + if not self._is_possible(pkg_name=dep_name): + continue - stack.append(dep_name) + stack.append(dep_name) real_packages = set(edges) if not transitive: @@ -257,7 +269,7 @@ def __init__( configuration: spack.config.Configuration, repo: spack.repo.RepoPath, store: spack.store.Store, - binary_index: spack.binary_distribution.BinaryCacheIndex, + binary_index: spack.binary_distribution.BinaryIndexCache, ): self.store = store self.binary_index = binary_index @@ -471,7 +483,7 @@ def possible_packages_facts(self, gen, fn): gen.newline() gen.h2("Packages with multiple possible nodes (build-tools)") - default = spack.config.CONFIG.get("concretizer:duplicates:max_dupes:default", 2) + default = spack.config.CONFIG.get("concretizer:duplicates:max_dupes:default", 1) duplicates = spack.config.CONFIG.get("concretizer:duplicates:max_dupes", {}) for package_name in sorted(self.possible_dependencies() & build_tools): max_dupes = duplicates.get(package_name, default) @@ -480,13 +492,8 @@ def possible_packages_facts(self, gen, fn): gen.fact(fn.multiple_unification_sets(package_name)) gen.newline() - gen.h2("Maximum number of nodes (link-run virtuals)") - for package_name in sorted(self._link_run_virtuals): - gen.fact(fn.max_dupes(package_name, 1)) - gen.newline() - - gen.h2("Maximum number of nodes (other virtuals)") - for package_name in sorted(self.possible_virtuals() - self._link_run_virtuals): + gen.h2("Maximum number of nodes (virtuals)") + for package_name in sorted(self.possible_virtuals()): max_dupes = duplicates.get(package_name, default) gen.fact(fn.max_dupes(package_name, max_dupes)) gen.newline() diff --git a/lib/spack/spack/solver/requirements.py b/lib/spack/spack/solver/requirements.py index 8f0de29f939d51..976ac263523205 100644 --- a/lib/spack/spack/solver/requirements.py +++ b/lib/spack/spack/solver/requirements.py @@ -2,19 +2,85 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import enum -from typing import List, NamedTuple, Optional, Sequence, Tuple +import warnings +from typing import List, NamedTuple, Optional, Sequence, Tuple, Union + +import spack.vendor.archspec.cpu import spack.config import spack.error import spack.package_base import spack.repo import spack.spec +import spack.spec_parser import spack.traverse +import spack.util.spack_yaml from spack.enums import PropagationPolicy from spack.llnl.util import tty from spack.util.spack_yaml import get_mark_from_yaml_data +def _mark_str(raw) -> str: + """Return a 'file:line: ' prefix from the YAML mark on *raw*, or empty string.""" + mark = get_mark_from_yaml_data(raw) + return f"{mark.name}:{mark.line + 1}: " if mark else "" + + +def _check_unknown_virtuals_on_edges(raw_strs: List[str], specs: List["spack.spec.Spec"]) -> None: + """Raise if any edge in *specs* requires a virtual that does not exist in the repository.""" + errors = [] + for raw, spec in zip(raw_strs, specs): + for edge in spack.traverse.traverse_edges([spec], root=False): + for virtual in edge.virtuals: + if not spack.repo.PATH.is_virtual(virtual): + errors.append( + f"{_mark_str(raw)}'{virtual}' in '{raw}' is not a known virtual package" + ) + if not errors: + return + if len(errors) == 1: + raise spack.error.InvalidVirtualOnEdgeError(errors[0]) + details = "\n".join(f" {idx}. {msg}" for idx, msg in enumerate(errors, 1)) + raise spack.error.InvalidVirtualOnEdgeError( + f"unknown virtuals have been detected in requirements:\n{details}" + ) + + +def _check_unknown_targets( + raw_strs: List[str], specs: List["spack.spec.Spec"], *, always_warn: bool = False +) -> None: + """Either warns or raises for unknown concrete target names in a set of specs. + + UserWarnings are emitted if *always_warn* is True or if there is at least one spec without + unknown targets. If all the specs have unknown targets raises an error. + """ + specs_with_unknown_targets = [ + (raw, spec) + for raw, spec in zip(raw_strs, specs) + if spec.architecture + and spec.architecture.target_concrete + and spec.target.name not in spack.vendor.archspec.cpu.TARGETS + ] + if not specs_with_unknown_targets: + return + + errors = [ + f"{_mark_str(raw)}'{spec}' contains unknown targets" + for raw, spec in specs_with_unknown_targets + ] + if len(errors) == 1: + msg = f"{errors[0]}. Run 'spack arch --known-targets' to see valid targets." + else: + details = "\n".join([f"{idx}. {part}" for idx, part in enumerate(errors, 1)]) + msg = ( + f"unknown targets have been detected in requirements\n{details}\n" + f"Run 'spack arch --known-targets' to see valid targets." + ) + if not always_warn and len(specs_with_unknown_targets) == len(specs): + raise spack.error.SpecError(msg) + warnings.warn(msg) + + class RequirementKind(enum.Enum): """Purpose / provenance of a requirement""" @@ -51,7 +117,7 @@ class RequirementRule(NamedTuple): def preference( pkg_name: str, constraint: spack.spec.Spec, - condition: spack.spec.Spec = spack.spec.Spec(), + condition: spack.spec.Spec = spack.spec.EMPTY_SPEC, origin: RequirementOrigin = RequirementOrigin.PREFER_YAML, kind: RequirementKind = RequirementKind.PACKAGE, message: Optional[str] = None, @@ -75,7 +141,7 @@ def preference( def conflict( pkg_name: str, constraint: spack.spec.Spec, - condition: spack.spec.Spec = spack.spec.Spec(), + condition: spack.spec.Spec = spack.spec.EMPTY_SPEC, origin: RequirementOrigin = RequirementOrigin.CONFLICT_YAML, kind: RequirementKind = RequirementKind.PACKAGE, message: Optional[str] = None, @@ -104,6 +170,14 @@ def __init__(self, configuration: spack.config.Configuration): self.runtime_pkgs = spack.repo.PATH.packages_with_tags("runtime") self.compiler_pkgs = spack.repo.PATH.packages_with_tags("compiler") self.preferences_from_input: List[Tuple[spack.spec.Spec, str]] = [] + self.toolchains = configuration.get_config("toolchains") + self._warned_compiler_all: set = set() + + def _parse_and_expand(self, string: str, *, named: bool = False) -> spack.spec.Spec: + result = parse_spec_from_yaml_string(string, named=named) + if self.toolchains: + spack.spec_parser.expand_toolchains(result, self.toolchains) + return result def rules(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]: result = [] @@ -175,6 +249,9 @@ def _rules_from_preferences( ) -> List[RequirementRule]: result = [] for item in preferences: + if kind == RequirementKind.DEFAULT: + # Warn about %gcc type of preferences under `all`. + self._maybe_warn_compiler_in_all(item, "prefer") spec, condition, msg = self._parse_prefer_conflict_item(item) result.append( preference(pkg_name, constraint=spec, condition=condition, kind=kind, message=msg) @@ -199,13 +276,16 @@ def _rules_from_conflicts( def _parse_prefer_conflict_item(self, item): # The item is either a string or an object with at least a "spec" attribute if isinstance(item, str): - spec = parse_spec_from_yaml_string(item) - condition = spack.spec.Spec() + spec = self._parse_and_expand(item) + condition = spack.spec.EMPTY_SPEC message = None else: - spec = parse_spec_from_yaml_string(item["spec"]) - condition = spack.spec.Spec(item.get("when")) + spec = self._parse_and_expand(item["spec"]) + when_str = item.get("when") + condition = self._parse_and_expand(when_str) if when_str else spack.spec.EMPTY_SPEC message = item.get("message") + raw_key = item if isinstance(item, str) else item.get("spec", item) + _check_unknown_targets([raw_key], [spec], always_warn=True) return spec, condition, message def _raw_yaml_data(self, pkg_name: str, *, section: str, virtual: bool = False): @@ -246,13 +326,20 @@ def _rules_from_requirements( constraints = [constraints] policy = "one_of" + if kind == RequirementKind.DEFAULT: + # Warn about %gcc type of requirements under `all`. + self._maybe_warn_compiler_in_all(constraints, "require") + # validate specs from YAML first, and fail with line numbers if parsing fails. + raw_strs = list(constraints) constraints = [ - parse_spec_from_yaml_string(constraint, named=kind == RequirementKind.VIRTUAL) - for constraint in constraints + self._parse_and_expand(constraint, named=kind == RequirementKind.VIRTUAL) + for constraint in raw_strs ] + _check_unknown_targets(raw_strs, constraints) + _check_unknown_virtuals_on_edges(raw_strs, constraints) when_str = requirement.get("when") - when = parse_spec_from_yaml_string(when_str) if when_str else spack.spec.Spec() + when = self._parse_and_expand(when_str) if when_str else spack.spec.EMPTY_SPEC constraints = [ x @@ -306,6 +393,59 @@ def reject_requirement_constraint( return True return False + def _maybe_warn_compiler_in_all(self, items: Union[str, list, dict], section: str) -> None: + """Warn once if a packages:all: prefer/require entry has compiler dependencies.""" + # Stick to single items, not complex one_of / any_of groups to keep things simple. + if isinstance(items, str): + spec_str = items + elif isinstance(items, dict) and "spec" in items and isinstance(items["spec"], str): + spec_str = items["spec"] + elif isinstance(items, list) and len(items) == 1 and isinstance(items[0], str): + spec_str = items[0] + else: + return + if spec_str in self._warned_compiler_all: + return + self._warned_compiler_all.add(spec_str) + suggestions = [] + for edge in self._parse_and_expand(spec_str).edges_to_dependencies(): + if edge.when != spack.spec.EMPTY_SPEC: + # Conditional dependencies are fine (includes toolchains after expansion). + continue + elif edge.virtuals: + # The case `%c,cxx=gcc` or similar. + keys = edge.virtuals + comment = "" + elif edge.spec.name in self.compiler_pkgs: + # Just a package `%gcc`. + keys = ("c",) + comment = "# For each language virtual (c, cxx, fortran, ...):\n" + else: + # Maybe %mpich or so? Just give a generic suggestion. + keys = ("",) + comment = "# For each virtual:\n" + data = {"packages": {k: {section: [str(edge.spec)]} for k in keys}} + suggestion = spack.util.spack_yaml.dump(data).rstrip() + suggestions.append(f"{comment}{suggestion}") + if suggestions: + mark = get_mark_from_yaml_data(spec_str) + location = f"{mark.name}:{mark.line + 1}: " if mark else "" + prefix = ( + f"{location}'packages: all: {section}: [\"{spec_str}\"]' applies a dependency " + f"constraint to all packages" + ) + suffix = "Consider instead:\n" + "\n".join(suggestions) + if section == "prefer": + warnings.warn( + f"{prefix}. This can lead to unexpected concretizations. This was likely " + f"intended as a preference for a provider of a (language) virtual. {suffix}" + ) + else: + warnings.warn( + f"{prefix}. This often leads to concretization errors. This was likely " + f"intended as a requirement for a provider of a (language) virtual. {suffix}" + ) + def _split_edge_on_virtuals(edge: spack.spec.DependencySpec) -> List[spack.spec.Spec]: """Split the edge on virtuals and removes the parent.""" diff --git a/lib/spack/spack/solver/reuse.py b/lib/spack/spack/solver/reuse.py index 81c2274f0b0362..8917a44904855a 100644 --- a/lib/spack/spack/solver/reuse.py +++ b/lib/spack/spack/solver/reuse.py @@ -3,11 +3,12 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import enum import functools -from typing import Any, Callable, List, Mapping +import typing +import warnings +from typing import Any, Callable, List, Mapping, Optional import spack.binary_distribution import spack.config -import spack.environment import spack.llnl.path import spack.repo import spack.spec @@ -19,116 +20,57 @@ complete_variants_and_architecture, extract_dicts_from_configuration, ) +from spack.spec_filter import SpecFilter from .runtimes import all_libcs +if typing.TYPE_CHECKING: + import spack.environment -class SpecFilter: - """Given a method to produce a list of specs, this class can filter them according to - different criteria. - """ - def __init__( - self, - factory: Callable[[], List[spack.spec.Spec]], - is_usable: Callable[[spack.spec.Spec], bool], - include: List[str], - exclude: List[str], - ) -> None: - """ - Args: - factory: factory to produce a list of specs - is_usable: predicate that takes a spec in input and returns False if the spec - should not be considered for this filter, True otherwise. - include: if present, a "good" spec must match at least one entry in the list - exclude: if present, a "good" spec must not match any entry in the list - """ - self.factory = factory - self.is_usable = is_usable - self.include = include - self.exclude = exclude - - def is_selected(self, s: spack.spec.Spec) -> bool: - if not self.is_usable(s): - return False - - if self.include and not any(s.satisfies(c) for c in self.include): - return False - - if self.exclude and any(s.satisfies(c) for c in self.exclude): - return False - - return True - - def selected_specs(self) -> List[spack.spec.Spec]: - return [s for s in self.factory() if self.is_selected(s)] - - @staticmethod - def from_store(configuration, *, packages_with_externals, include, exclude) -> "SpecFilter": - """Constructs a filter that takes the specs from the current store.""" - is_reusable = functools.partial( - _is_reusable, packages_with_externals=packages_with_externals, local=True - ) - factory = functools.partial(_specs_from_store, configuration=configuration) - return SpecFilter(factory=factory, is_usable=is_reusable, include=include, exclude=exclude) - - @staticmethod - def from_buildcache(*, packages_with_externals, include, exclude) -> "SpecFilter": - """Constructs a filter that takes the specs from the configured buildcaches.""" - is_reusable = functools.partial( - _is_reusable, packages_with_externals=packages_with_externals, local=False - ) - return SpecFilter( - factory=_specs_from_mirror, is_usable=is_reusable, include=include, exclude=exclude - ) +def spec_filter_from_store( + configuration, *, packages_with_externals, include, exclude +) -> SpecFilter: + """Constructs a filter that takes the specs from the current store.""" + is_reusable = functools.partial( + _is_reusable, packages_with_externals=packages_with_externals, local=True + ) + factory = functools.partial(_specs_from_store, configuration=configuration) + return SpecFilter(factory=factory, is_usable=is_reusable, include=include, exclude=exclude) - @staticmethod - def from_environment(*, packages_with_externals, include, exclude, env) -> "SpecFilter": - is_reusable = functools.partial( - _is_reusable, packages_with_externals=packages_with_externals, local=True - ) - factory = functools.partial(_specs_from_environment, env=env) - return SpecFilter(factory=factory, is_usable=is_reusable, include=include, exclude=exclude) - @staticmethod - def from_environment_included_concrete( - *, - packages_with_externals, - include: List[str], - exclude: List[str], - env: spack.environment.Environment, - included_concrete: str, - ) -> "SpecFilter": - is_reusable = functools.partial( - _is_reusable, packages_with_externals=packages_with_externals, local=True - ) - factory = functools.partial( - _specs_from_environment_included_concrete, env=env, included_concrete=included_concrete - ) - return SpecFilter(factory=factory, is_usable=is_reusable, include=include, exclude=exclude) - - @staticmethod - def from_packages_yaml( - *, external_parser: ExternalSpecsParser, packages_with_externals, include, exclude - ) -> "SpecFilter": - is_reusable = functools.partial( - _is_reusable, packages_with_externals=packages_with_externals, local=True - ) - return SpecFilter( - external_parser.all_specs, is_usable=is_reusable, include=include, exclude=exclude - ) +def spec_filter_from_buildcache(*, packages_with_externals, include, exclude) -> SpecFilter: + """Constructs a filter that takes the specs from the configured buildcaches.""" + is_reusable = functools.partial( + _is_reusable, packages_with_externals=packages_with_externals, local=False + ) + return SpecFilter( + factory=_specs_from_mirror, is_usable=is_reusable, include=include, exclude=exclude + ) -def _has_runtime_dependencies(spec: spack.spec.Spec) -> bool: - # TODO (compiler as nodes): this function contains specific names from builtin, and should - # be made more general - if "gcc" in spec and "gcc-runtime" not in spec: - return False +def spec_filter_from_environment(*, packages_with_externals, include, exclude, env) -> SpecFilter: + is_reusable = functools.partial( + _is_reusable, packages_with_externals=packages_with_externals, local=True + ) + factory = functools.partial(_specs_from_environment, env=env) + return SpecFilter(factory=factory, is_usable=is_reusable, include=include, exclude=exclude) + + +def spec_filter_from_packages_yaml( + *, external_parser: ExternalSpecsParser, packages_with_externals, include, exclude +) -> SpecFilter: + is_reusable = functools.partial( + _is_reusable, packages_with_externals=packages_with_externals, local=True + ) + return SpecFilter( + external_parser.all_specs, is_usable=is_reusable, include=include, exclude=exclude + ) - if "intel-oneapi-compilers" in spec and "intel-oneapi-runtime" not in spec: - return False - return True +def _has_runtime_dependencies(spec: spack.spec.Spec) -> bool: + # Spack v1.0 specs and later + return spec.original_spec_format() >= 5 def _is_reusable(spec: spack.spec.Spec, packages_with_externals, local: bool) -> bool: @@ -188,12 +130,15 @@ def _specs_from_store(configuration): def _specs_from_mirror(): try: - return spack.binary_distribution.update_cache_and_get_specs() + specs = spack.binary_distribution.update_cache_and_get_specs() except (spack.binary_distribution.FetchCacheError, IndexError): # this is raised when no mirrors had indices. # TODO: update mirror configuration so it can indicate that the # TODO: source cache (or any mirror really) doesn't have binaries. return [] + for url in sorted(spack.binary_distribution.BINARY_INDEX.mirrors_without_index): + warnings.warn(f"the mirror at {url} cannot be used in concretization (no index found)") + return specs def _specs_from_environment(env): @@ -204,15 +149,6 @@ def _specs_from_environment(env): return [] -def _specs_from_environment_included_concrete(env, included_concrete): - """Return only concrete specs from the environment included from the included_concrete""" - if env: - assert included_concrete in env.included_concrete_envs - return [concrete for concrete in env.included_specs_by_hash[included_concrete].values()] - else: - return [] - - class ReuseStrategy(enum.Enum): ROOTS = enum.auto() DEPENDENCIES = enum.auto() @@ -235,24 +171,40 @@ def create_external_parser( return ExternalSpecsParser(external_dicts, complete_node=complete_fn) +SpecFiltersFactory = Callable[ + [Callable[[spack.spec.Spec], bool], spack.config.Configuration], List[SpecFilter] +] + + class ReusableSpecsSelector: """Selects specs that can be reused during concretization.""" def __init__( self, + *, configuration: spack.config.Configuration, external_parser: ExternalSpecsParser, packages_with_externals: Any, + factory: Optional[SpecFiltersFactory] = None, ) -> None: + # Local import to break circular dependencies + import spack.environment + self.configuration = configuration self.store = spack.store.create(configuration) self.reuse_strategy = ReuseStrategy.ROOTS - reuse_yaml = self.configuration.get("concretizer:reuse", False) + self.reuse_sources = [] + if factory is not None: + is_reusable = functools.partial( + _is_reusable, packages_with_externals=packages_with_externals, local=True + ) + self.reuse_sources.extend(factory(is_reusable, configuration)) + if not isinstance(reuse_yaml, Mapping): self.reuse_sources.append( - SpecFilter.from_packages_yaml( + spec_filter_from_packages_yaml( external_parser=external_parser, packages_with_externals=packages_with_externals, include=[], @@ -267,21 +219,15 @@ def __init__( self.reuse_strategy = ReuseStrategy.DEPENDENCIES self.reuse_sources.extend( [ - SpecFilter.from_store( + spec_filter_from_store( configuration=self.configuration, packages_with_externals=packages_with_externals, include=[], exclude=[], ), - SpecFilter.from_buildcache( + spec_filter_from_buildcache( packages_with_externals=packages_with_externals, include=[], exclude=[] ), - SpecFilter.from_environment( - packages_with_externals=packages_with_externals, - include=[], - exclude=[], - env=spack.environment.active_environment(), # with all concrete includes - ), ] ) else: @@ -300,45 +246,20 @@ def __init__( if source["type"] == "environment" and "path" in source: env_dir = spack.environment.as_env_dir(source["path"]) active_env = spack.environment.active_environment() - if active_env and env_dir in active_env.included_concrete_envs: - # If the environment is included as a concrete environment, use the - # local copy of specs in the active environment. - # note: included concrete environments are only updated at concretization - # time, and reuse needs to match the included specs. - self.reuse_sources.append( - SpecFilter.from_environment_included_concrete( - packages_with_externals=packages_with_externals, - include=include, - exclude=exclude, - env=active_env, - included_concrete=env_dir, - ) - ) - else: + if not active_env or env_dir not in active_env.included_concrete_env_root_dirs: # If the environment is not included as a concrete environment, use the # current specs from its lockfile. self.reuse_sources.append( - SpecFilter.from_environment( + spec_filter_from_environment( packages_with_externals=packages_with_externals, include=include, exclude=exclude, env=spack.environment.environment_from_name_or_dir(env_dir), ) ) - elif source["type"] == "environment": - # reusing from the current environment implicitly reuses from all of the - # included concrete environments - self.reuse_sources.append( - SpecFilter.from_environment( - packages_with_externals=packages_with_externals, - include=include, - exclude=exclude, - env=spack.environment.active_environment(), - ) - ) elif source["type"] == "local": self.reuse_sources.append( - SpecFilter.from_store( + spec_filter_from_store( self.configuration, packages_with_externals=packages_with_externals, include=include, @@ -347,7 +268,7 @@ def __init__( ) elif source["type"] == "buildcache": self.reuse_sources.append( - SpecFilter.from_buildcache( + spec_filter_from_buildcache( packages_with_externals=packages_with_externals, include=include, exclude=exclude, @@ -359,7 +280,7 @@ def __init__( # Since libcs are implicit externals, we need to implicitly include them include = include + sorted(all_libcs()) # type: ignore[type-var] self.reuse_sources.append( - SpecFilter.from_packages_yaml( + spec_filter_from_packages_yaml( external_parser=external_parser, packages_with_externals=packages_with_externals, include=include, @@ -370,7 +291,7 @@ def __init__( # If "external" is not specified, we assume that all externals have to be included if not has_external_source: self.reuse_sources.append( - SpecFilter.from_packages_yaml( + spec_filter_from_packages_yaml( external_parser=external_parser, packages_with_externals=packages_with_externals, include=[], diff --git a/lib/spack/spack/solver/runtimes.py b/lib/spack/spack/solver/runtimes.py index db54085c350faa..2b9c52619c9e1a 100644 --- a/lib/spack/spack/solver/runtimes.py +++ b/lib/spack/spack/solver/runtimes.py @@ -79,7 +79,7 @@ def depends_on(self, dependency_str: str, *, when: str, type: str, description: dependency_spec = spack.spec.Spec(dependency_str) if dependency_spec.versions != spack.version.any_version: - self._setup.version_constraints.add((dependency_spec.name, dependency_spec.versions)) + self._setup.version_constraints[dependency_spec.name].add(dependency_spec.versions) self.injected_dependencies.add(dependency_spec) body_str, node_variable = self.rule_body_from(when_spec) @@ -115,7 +115,7 @@ def depends_on(self, dependency_str: str, *, when: str, type: str, description: f" provider(ProviderNode, {runtime_node}),\n" ) - rule = f"{head_str} :-\n" f"{depends_on_constraint}" f"{body_str}." + rule = f"{head_str} :-\n{depends_on_constraint}{body_str}." self.rules.append(rule) self.reset() @@ -134,8 +134,7 @@ def rule_body_from(self, when_spec: "spack.spec.Spec") -> Tuple[str, str]: when_substitutions = {} for s in when_spec.traverse(root=False): when_substitutions[f'"{s.name}"'] = self.node_for(s.name) - when_spec.name = node_placeholder - body_clauses = self._setup.spec_clauses(when_spec, body=True) + body_clauses = self._setup.spec_clauses(when_spec, name=node_placeholder, body=True) for clause in body_clauses: if clause.args[0] == "virtual_on_incoming_edges": # Substitute: attr("virtual_on_incoming_edges", ProviderNode, Virtual) @@ -196,9 +195,7 @@ def propagate(self, constraint_str: str, *, when: str): constraint_clauses = self._setup.spec_clauses(constraint_spec, body=False) for clause in constraint_clauses: if clause.args[0] == "node_version_satisfies": - self._setup.version_constraints.add( - (constraint_spec.name, constraint_spec.versions) - ) + self._setup.version_constraints[constraint_spec.name].add(constraint_spec.versions) args = f'"{constraint_spec.name}", "{constraint_spec.versions}"' head_str = f"propagate({node_variable}, node_version_satisfies({args}))" rule = f"{head_str} :-\n{body_str}." diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 26929412d2d30a..fab65cba4a46fa 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -46,6 +46,7 @@ 6. The architecture to build with. """ + import collections import collections.abc import enum @@ -100,6 +101,7 @@ import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml import spack.variant as vt +import spack.version import spack.version as vn import spack.version.git_ref_lookup @@ -122,7 +124,6 @@ "SpecDeprecatedError", ] - SPEC_FORMAT_RE = re.compile( r"(?:" # this is one big or, with matches ordered by priority # OPTION 1: escaped character (needs to be first to catch opening \{) @@ -758,7 +759,7 @@ def __init__( self.virtuals = tuple(sorted(set(virtuals))) self.direct = direct self.propagation = propagation - self.when = when or Spec() + self.when = when or EMPTY_SPEC def update_deptypes(self, depflag: dt.DepFlag) -> bool: """Update the current dependency types""" @@ -795,6 +796,26 @@ def copy(self, *, keep_virtuals: bool = True, keep_parent: bool = True) -> "Depe when=self.when, ) + def _constrain(self, other: "DependencySpec") -> bool: + """Constrain this edge with another edge. Precondition: parent and child of self and other + are compatible, and both edges have the same when condition. Used as an internal helper + function in Spec.constrain. + + Args: + other: edge to use as constraint + + Returns: + True if the current edge was changed, False otherwise. + """ + changed = False + changed |= self.spec.constrain(other.spec) + changed |= self.update_deptypes(other.depflag) + changed |= self.update_virtuals(other.virtuals) + if not self.direct and other.direct: + changed = True + self.direct = True + return changed + def _cmp_iter(self): yield self.parent.name if self.parent else None yield self.spec.name if self.spec else None @@ -846,7 +867,7 @@ def format(self, *, unconditional: bool = False) -> str: return f"{parent_str} {dep_sigil}{child_str}" def flip(self) -> "DependencySpec": - """Flips the dependency and keeps its type. Drops all othe information.""" + """Flips the dependency and keeps its type. Drops all other information.""" return DependencySpec( parent=self.spec, spec=self.parent, depflag=self.depflag, virtuals=() ) @@ -908,12 +929,6 @@ def _shared_subset_pair_iterate(container1, container2): class FlagMap(lang.HashableMap[str, List[CompilerFlag]]): - __slots__ = ("spec",) - - def __init__(self, spec): - super().__init__() - self.spec = spec - def satisfies(self, other): return all(f in self and set(self[f]) >= set(other[f]) for f in other) @@ -955,7 +970,7 @@ def valid_compiler_flags(): return _valid_compiler_flags def copy(self): - clone = FlagMap(self.spec) + clone = FlagMap() for name, compiler_flag in self.items(): clone[name] = compiler_flag return clone @@ -992,7 +1007,7 @@ def yaml_entry(self, flag_type): return flag_type, [str(flag) for flag in self[flag_type]] def _cmp_iter(self): - for k, v in sorted(self.items()): + for k, v in sorted(self.dict.items()): yield k def flags(): @@ -1294,47 +1309,6 @@ def __set__(self, instance, value): QueryState = collections.namedtuple("QueryState", ["name", "extra_parameters", "isvirtual"]) -class SpecBuildInterface(lang.ObjectWrapper): - # home is available in the base Package so no default is needed - home = ForwardQueryToPackage("home", default_handler=None) - headers = ForwardQueryToPackage("headers", default_handler=_headers_default_handler) - libs = ForwardQueryToPackage("libs", default_handler=_libs_default_handler) - command = ForwardQueryToPackage("command", default_handler=None, _indirect=True) - - def __init__( - self, - spec: "Spec", - name: str, - query_parameters: List[str], - _parent: "Spec", - is_virtual: bool, - ): - super().__init__(spec) - # Adding new attributes goes after super() call since the ObjectWrapper - # resets __dict__ to behave like the passed object - original_spec = getattr(spec, "wrapped_obj", spec) - self.wrapped_obj = original_spec - self.token = original_spec, name, query_parameters, _parent, is_virtual - self.last_query = QueryState( - name=name, extra_parameters=query_parameters, isvirtual=is_virtual - ) - - # TODO: this ad-hoc logic makes `spec["python"].command` return - # `spec["python-venv"].command` and should be removed when `python` is a virtual. - self.indirect_spec = None - if spec.name == "python": - python_venvs = _parent.dependencies("python-venv") - if not python_venvs: - return - self.indirect_spec = python_venvs[0] - - def __reduce__(self): - return SpecBuildInterface, self.token - - def copy(self, *args, **kwargs): - return self.wrapped_obj.copy(*args, **kwargs) - - def tree( specs: List["Spec"], *, @@ -1503,6 +1477,83 @@ def _anonymous_star(dep: DependencySpec, dep_format: str) -> str: return "*" if dep.spec.architecture else "" +def _get_satisfying_edge( + lhs_node: "Spec", rhs_edge: DependencySpec, *, resolve_virtuals: bool +) -> Optional[DependencySpec]: + """Search for an edge in ``lhs_node`` that satisfies ``rhs_edge``.""" + # First check direct deps of all types. + for lhs_edge in lhs_node.edges_to_dependencies(): + if _satisfies_edge(lhs_edge, rhs_edge, resolve_virtuals): + return lhs_edge + + # Include the historical compiler node if available as an ad-hoc edge. + compiler_spec = lhs_node.annotations.compiler_node_attribute + if compiler_spec is not None: + compiler_edge = DependencySpec( + lhs_node, + compiler_spec, + depflag=dt.BUILD, + virtuals=("c", "cxx", "fortran"), + direct=True, + ) + if _satisfies_edge(compiler_edge, rhs_edge, resolve_virtuals): + return compiler_edge + + if rhs_edge.direct: + return None + + # BFS through link/run transitive deps (skip depth 1, already checked). + depflag = dt.LINK | dt.RUN + queue = collections.deque(lhs_node.edges_to_dependencies(depflag=depflag)) + seen = {id(lhs_edge.spec) for lhs_edge in queue} + while queue: + lhs_edge = queue.popleft() + + if _satisfies_edge(lhs_edge, rhs_edge, resolve_virtuals): + return lhs_edge + + for lhs_edge in lhs_edge.spec.edges_to_dependencies(depflag=depflag): + if id(lhs_edge.spec) not in seen: + seen.add(id(lhs_edge.spec)) + queue.append(lhs_edge) + + return None + + +def _satisfies_edge(lhs: "DependencySpec", rhs: "DependencySpec", resolve_virtuals: bool) -> bool: + """Helper function for satisfaction tests, which checks edge attributes and the target node. + It skips verification of the parent node.""" + name_mismatch = rhs.spec.name and lhs.spec.name != rhs.spec.name + if name_mismatch and rhs.spec.name not in lhs.virtuals: + return False + + if not rhs.when._satisfies(lhs.when, resolve_virtuals=resolve_virtuals): + return False + + # Subset semantics for virtuals + for v in rhs.virtuals: + if v not in lhs.virtuals: + return False + + # Subset semantics for dependency types + if (lhs.depflag & rhs.depflag) != rhs.depflag: + return False + + if not name_mismatch: + return lhs.spec._satisfies_node(rhs.spec, resolve_virtuals=resolve_virtuals) + + # Right-hand side is virtual provided by left-hand side. The only node attribute supported is + # the version of the virtual. Avoid expensive lookups for provider metadata if there's no + # version constraint to check. + if rhs.spec.versions == spack.version.any_version: + return True + + if not resolve_virtuals: + return False + + return lhs.spec._provides_virtual(rhs.spec) + + @lang.lazy_lexicographic_ordering(set_hash=False) class Spec: compiler = DeprecatedCompilerSpec() @@ -1523,8 +1574,7 @@ def __init__(self, spec_like=None, *, external_path=None, external_modules=None) Keyword arguments: external_path: prefix, if this is a spec for an external package - external_modules: list of external modules, if this is an external package - using modules. + external_modules: list of external modules, for an external package using modules """ # Copy if spec_like is a Spec. if isinstance(spec_like, Spec): @@ -1534,9 +1584,9 @@ def __init__(self, spec_like=None, *, external_path=None, external_modules=None) # init an empty spec that matches anything. self.name: str = "" self.versions = vn.VersionList.any() - self.variants = VariantMap(self) + self.variants = VariantMap() self.architecture = None - self.compiler_flags = FlagMap(self) + self.compiler_flags = FlagMap() self._dependents = {} self._dependencies = {} self.namespace = None @@ -1843,7 +1893,7 @@ def _add_dependency( when: optional condition under which dependency holds """ if when is None: - when = Spec() + when = EMPTY_SPEC if spec.name not in self._dependencies or not spec.name: self.add_dependency_edge( @@ -1914,7 +1964,7 @@ def add_dependency_edge( when: if non-None, condition under which dependency holds """ if when is None: - when = Spec() + when = EMPTY_SPEC # Check if we need to update edges that are already present selected = self._dependencies.get(dependency_spec.name, []) @@ -2577,8 +2627,11 @@ def node_dict_with_hashes(self, hash: ht.SpecHashDescriptor = ht.dag_hash) -> Di def to_yaml(self, stream=None, hash=ht.dag_hash): return syaml.dump(self.to_dict(hash), stream=stream, default_flow_style=False) - def to_json(self, stream=None, hash=ht.dag_hash): - return sjson.dump(self.to_dict(hash), stream) + def to_json(self, stream=None, *, hash=ht.dag_hash, pretty=False): + if stream is None: + return sjson.dumps(self.to_dict(hash), pretty=pretty) + sjson.dump(self.to_dict(hash), stream, pretty=pretty) + return None @staticmethod def from_specfile(path): @@ -2859,9 +2912,9 @@ def _patches_assigned(self): # ensure that patch state is consistent patch_variant = self.variants["patches"] - assert hasattr( - patch_variant, "_patches_in_order_of_appearance" - ), "patches should always be assigned with a patch variant." + assert hasattr(patch_variant, "_patches_in_order_of_appearance"), ( + "patches should always be assigned with a patch variant." + ) return True @@ -3136,7 +3189,7 @@ def _constrain(self, other, deps=True, *, resolve_virtuals: bool): changed = True changed |= self.versions.intersect(other.versions) - changed |= self.variants.constrain(other.variants) + changed |= self._constrain_variants(other) changed |= self.compiler_flags.constrain(other.compiler_flags) @@ -3166,29 +3219,28 @@ def _constrain_dependencies(self, other: "Spec", resolve_virtuals: bool = True) if not other._intersects_dependencies(self, resolve_virtuals=resolve_virtuals): raise UnsatisfiableDependencySpecError(other, self) - if any(not d.name for d in other.traverse(root=False)): - raise UnconstrainableDependencySpecError(other) - - reference_spec = self.copy(deps=True) - for edge in other.edges_to_dependencies(): - existing = [ - e for e in self.edges_to_dependencies(edge.spec.name) if e.when == edge.when - ] - if existing: - existing[0].spec.constrain(edge.spec) - existing[0].update_deptypes(edge.depflag) - existing[0].update_virtuals(edge.virtuals) - existing[0].direct |= edge.direct + for d in other.traverse(root=False): + if not d.name: + raise UnconstrainableDependencySpecError(other) + changed = False + for other_edge in other.edges_to_dependencies(): + # Find the first edge in self that matches other_edge by name and when clause. + for self_edge in self.edges_to_dependencies(other_edge.spec.name): + if self_edge.when == other_edge.when: + changed |= self_edge._constrain(other_edge) + break else: + # Otherwise, a copy of the edge is added as a constraint to self. + changed = True self.add_dependency_edge( - edge.spec, - depflag=edge.depflag, - virtuals=edge.virtuals, - direct=edge.direct, - propagation=edge.propagation, - when=edge.when, + other_edge.spec.copy(deps=True), + depflag=other_edge.depflag, + virtuals=other_edge.virtuals, + direct=other_edge.direct, + propagation=other_edge.propagation, + when=other_edge.when, # no need to copy; when conditions are immutable ) - return self != reference_spec + return changed def constrained(self, other, deps=True): """Return a constrained copy without modifying this spec.""" @@ -3222,6 +3274,8 @@ def intersects(self, other: Union[str, "Spec"], deps: bool = True) -> bool: def _intersects( self, other: Union[str, "Spec"], deps: bool = True, resolve_virtuals: bool = True ) -> bool: + if other is EMPTY_SPEC: + return True other = self._autospec(other) if other.concrete and self.concrete: @@ -3289,7 +3343,7 @@ def _intersects( if not self.versions.intersects(other.versions): return False - if not self.variants.intersects(other.variants): + if not self._intersects_variants(other): return False if self.architecture and other.architecture: @@ -3360,6 +3414,42 @@ def satisfies(self, other: Union[str, "Spec"], deps: bool = True) -> bool: """ return self._satisfies(other=other, deps=deps, resolve_virtuals=True) + def _provides_virtual(self, virtual_spec: "Spec") -> bool: + """Return True if this spec provides the given virtual spec. + + Args: + virtual_spec: abstract virtual spec (e.g. ``"mpi"`` or ``"mpi@3:"``) + """ + if not virtual_spec.name: + return False + + # Get the package instance + if self.concrete: + try: + pkg = self.package + except spack.repo.UnknownPackageError: + return False + else: + try: + pkg_cls = spack.repo.PATH.get_pkg_class(self.fullname) + pkg = pkg_cls(self) + except spack.repo.UnknownEntityError: + # If we can't get package info on this spec, don't treat + # it as a provider of this vdep. + return False + + for when_spec, provided in pkg.provided.items(): + # Don't use satisfies for virtuals, because an abstract vs. abstract spec may use the + # repo index + if self.satisfies(when_spec, deps=False) and any( + provided_virtual.name == virtual_spec.name + and provided_virtual.versions.intersects(virtual_spec.versions) + for provided_virtual in provided + ): + return True + + return False + def _satisfies( self, other: Union[str, "Spec"], deps: bool = True, resolve_virtuals: bool = True ) -> bool: @@ -3371,14 +3461,55 @@ def _satisfies( resolve_virtuals: if True, resolve virtuals in self and other. This requires a repository to be available. """ + if other is EMPTY_SPEC: + return True + other = self._autospec(other) + if not self._satisfies_node(other, resolve_virtuals=resolve_virtuals): + return False + + # If there are no dependencies on the rhs, or we don't recurse, they are satisfied. + if not deps or not other._dependencies: + return True + + stack = [(self, other)] + + while stack: + lhs, rhs = stack.pop() + + for rhs_edge in rhs.edges_to_dependencies(): + # Skip rhs edges whose when condition doesn't apply to the lhs node. + if rhs_edge.when is not EMPTY_SPEC and not lhs._intersects( + rhs_edge.when, resolve_virtuals=resolve_virtuals + ): + continue + + lhs_edge = _get_satisfying_edge(lhs, rhs_edge, resolve_virtuals=resolve_virtuals) + + if not lhs_edge: + return False + + # Recursive case: `^zlib %gcc` + if not rhs_edge.spec.concrete and rhs_edge.spec._dependencies: + stack.append((lhs_edge.spec, rhs_edge.spec)) + + return True + + def _satisfies_node(self, other: "Spec", resolve_virtuals: bool) -> bool: + """Compares self and other without looking at dependencies""" if other.concrete: # The left-hand side must be the same singleton with identical hash. Notice that # package hashes can be different for otherwise indistinguishable concrete Spec # objects. return self.concrete and self.dag_hash() == other.dag_hash() + if self.name != other.name and self.name and other.name: + # Name mismatch can still be satisfiable if lhs provides the virtual mentioned by rhs. + if not resolve_virtuals: + return False + return self._provides_virtual(other) + # If the right-hand side has an abstract hash, make sure it's a prefix of the # left-hand side's (abstract) hash. if other.abstract_hash: @@ -3386,28 +3517,6 @@ def _satisfies( if not compare_hash or not compare_hash.startswith(other.abstract_hash): return False - # If the names are different, we need to consider virtuals - if self.name != other.name and self.name and other.name and resolve_virtuals: - # A concrete provider can satisfy a virtual dependency. - if not spack.repo.PATH.is_virtual(self.name) and spack.repo.PATH.is_virtual( - other.name - ): - try: - # Here we might get an abstract spec - pkg_cls = spack.repo.PATH.get_pkg_class(self.fullname) - pkg = pkg_cls(self) - except spack.repo.UnknownEntityError: - # If we can't get package info on this spec, don't treat - # it as a provider of this vdep. - return False - - if pkg.provides(other.name): - for when_spec, provided in pkg.provided.items(): - if self.satisfies(when_spec, deps=False): - if any(vpkg.intersects(other) for vpkg in provided): - return True - return False - # namespaces either match, or other doesn't require one. if ( other.namespace is not None @@ -3419,7 +3528,7 @@ def _satisfies( if not self.versions.satisfies(other.versions): return False - if not self.variants.satisfies(other.variants): + if not self._satisfies_variants(other): return False if self.architecture and other.architecture: @@ -3431,170 +3540,95 @@ def _satisfies( if not self.compiler_flags.satisfies(other.compiler_flags): return False - # If we need to descend into dependencies, do it, otherwise we're done. - if not deps: - return True - - # If there are no constraints to satisfy, we're done. - if not other._dependencies: - return True - - # If we arrived here, the lhs root node satisfies the rhs root node. Now we need to check - # all the edges that have an abstract parent, and verify that they match some edge in the - # lhs. - # - # It might happen that the rhs brings in concrete sub-DAGs. For those we don't need to - # verify the edge properties, cause everything is encoded in the hash of the nodes that - # will be verified later. - lhs_edges: Dict[str, Set[DependencySpec]] = collections.defaultdict(set) - mock_nodes_from_old_specfiles = set() - for rhs_edge in other.traverse_edges(root=False, cover="edges"): - # Check satisfaction of the dependency only if its when condition can apply - if not rhs_edge.parent.name or rhs_edge.parent.name == self.name: - test_spec = self - elif rhs_edge.parent.name in self: - test_spec = self[rhs_edge.parent.name] - else: - test_spec = None - if test_spec and not test_spec._intersects( - rhs_edge.when, resolve_virtuals=resolve_virtuals - ): - continue - - # If we are checking for ^mpi we need to verify if there is any edge - if resolve_virtuals and spack.repo.PATH.is_virtual(rhs_edge.spec.name): - # Don't mutate objects in memory that may be referred elsewhere - rhs_edge = rhs_edge.copy() - rhs_edge.update_virtuals(virtuals=(rhs_edge.spec.name,)) - - if rhs_edge.direct: - # Note: this relies on abstract specs from string not being deeper than 2 levels - # e.g. in foo %fee ^bar %baz we cannot go deeper than "baz" and e.g. specify its - # dependencies too. - # - # We also need to account for cases like gcc@ %gcc@ where the parent - # name is the same as the child name - # - # The same assumptions hold on Spec.constrain, and Spec.intersect - current_node = self - if rhs_edge.parent.name and rhs_edge.parent.name != rhs_edge.spec.name: - try: - current_node = self[rhs_edge.parent.name] - except KeyError: - return False + return True - if current_node.original_spec_format() < 5 or ( - # If the current external node has dependencies, it has no annotations - current_node.original_spec_format() >= 5 - and current_node.external - and not current_node._dependencies - ): - compiler_spec = current_node.annotations.compiler_node_attribute - if compiler_spec is None: - return False - - mock_nodes_from_old_specfiles.add(compiler_spec) - # This checks that the single node compiler spec satisfies the request - # of a direct dependency. The check is not perfect, but based on heuristic. - if not compiler_spec._satisfies( - rhs_edge.spec, resolve_virtuals=resolve_virtuals - ): - return False + def _satisfies_variants(self, other: "Spec") -> bool: + if self.concrete: + return self._satisfies_variants_when_self_concrete(other) + return self._satisfies_variants_when_self_abstract(other) - else: - # If the branch is % or ^, check if we have a corresponding - # branch in the lhs - candidate_edges = [] - if resolve_virtuals and spack.repo.PATH.is_virtual(rhs_edge.spec.name): - candidate_edges = current_node.edges_to_dependencies( - name=rhs_edge.spec.name - ) + def _satisfies_variants_when_self_concrete(self, other: "Spec") -> bool: + non_propagating, propagating = other.variants.partition_variants() + result = all( + name in self.variants and self.variants[name].satisfies(other.variants[name]) + for name in non_propagating + ) + if not propagating: + return result - name = ( - None - if resolve_virtuals and spack.repo.PATH.is_virtual(rhs_edge.spec.name) - else rhs_edge.spec.name - ) - candidate_edges.extend( - current_node.edges_to_dependencies( - name=name, virtuals=rhs_edge.virtuals or None - ) - ) - # Select at least the deptypes on the rhs_edge, and conditional edges that - # constrain a bigger portion of the search space (so it's rhs.when <= lhs.when) - candidates = [ - lhs_edge.spec - for lhs_edge in candidate_edges - if ((lhs_edge.depflag & rhs_edge.depflag) ^ rhs_edge.depflag) == 0 - and rhs_edge.when._satisfies( - lhs_edge.when, resolve_virtuals=resolve_virtuals - ) - ] - if not candidates or not any( - x._satisfies(rhs_edge.spec, resolve_virtuals=resolve_virtuals) - for x in candidates - ): - return False + for node in self.traverse(): + if not all( + node.variants[name].satisfies(other.variants[name]) + for name in propagating + if name in node.variants + ): + return False + return result - continue + def _satisfies_variants_when_self_abstract(self, other: "Spec") -> bool: + other_non_propagating, other_propagating = other.variants.partition_variants() + self_non_propagating, self_propagating = self.variants.partition_variants() - # Skip edges from a concrete sub-DAG - if rhs_edge.parent.concrete: - continue + # First check variants without propagation set + result = all( + name in self_non_propagating + and ( + self.variants[name].propagate + or self.variants[name].satisfies(other.variants[name]) + ) + for name in other_non_propagating + ) + if result is False or (not other_propagating and not self_propagating): + return result - if not lhs_edges: - # Construct a map of the link/run subDAG + direct "build" edges, - # keyed by dependency name - for lhs_edge in self.traverse_edges( - root=False, cover="edges", deptype=("link", "run") + # Check that self doesn't contradict variants propagated by other + if other_propagating: + for node in self.traverse(): + if not all( + node.variants[name].satisfies(other.variants[name]) + for name in other_propagating + if name in node.variants ): - lhs_edges[lhs_edge.spec.name].add(lhs_edge) - for virtual_name in lhs_edge.virtuals: - lhs_edges[virtual_name].add(lhs_edge) - - build_edges = self.edges_to_dependencies(depflag=dt.BUILD) - for lhs_edge in build_edges: - lhs_edges[lhs_edge.spec.name].add(lhs_edge) - for virtual_name in lhs_edge.virtuals: - lhs_edges[virtual_name].add(lhs_edge) - - # We don't have edges to this dependency - current_dependency_name = rhs_edge.spec.name - if current_dependency_name and current_dependency_name not in lhs_edges: - return False + return False - if not current_dependency_name: - # Here we have an anonymous spec e.g. ^ dev_path=* - candidate_edges = list(itertools.chain(*lhs_edges.values())) + # Check that other doesn't contradict variants propagated by self + if self_propagating: + for node in other.traverse(): + if not all( + node.variants[name].satisfies(self.variants[name]) + for name in self_propagating + if name in node.variants + ): + return False - else: - candidate_edges = [ - lhs_edge - for lhs_edge in lhs_edges[current_dependency_name] - if rhs_edge.when._satisfies(lhs_edge.when, resolve_virtuals=resolve_virtuals) - ] + return result - if not candidate_edges: - return False + def _intersects_variants(self, other: "Spec") -> bool: + self_dict = self.variants.dict + other_dict = other.variants.dict + return all(self_dict[k].intersects(other_dict[k]) for k in other_dict if k in self_dict) - for virtual in rhs_edge.virtuals: - # Check the name because ^mpi has the "mpi" virtual - has_virtual = any( - virtual in edge.virtuals or virtual == edge.spec.name - for edge in candidate_edges - ) - if not has_virtual: - return False + def _constrain_variants(self, other: "Spec") -> bool: + """Add all variants in other that aren't in self to self. Also constrain all multi-valued + variants that are already present. Return True iff self changed""" + if other is not None and other._concrete: + for k in self.variants: + if k not in other.variants: + raise vt.UnsatisfiableVariantSpecError(self.variants[k], "") - for lhs_edge in candidate_edges: - if lhs_edge.spec._satisfies( - rhs_edge.spec, deps=False, resolve_virtuals=resolve_virtuals - ): - break + changed = False + for k in other.variants: + if k in self.variants: + if not self.variants[k].intersects(other.variants[k]): + raise vt.UnsatisfiableVariantSpecError(self.variants[k], other.variants[k]) + # If they are compatible merge them + changed |= self.variants[k].constrain(other.variants[k]) else: - return False + # If it is not present copy it straight away + self.variants[k] = other.variants[k].copy() + changed = True - return True + return changed @property # type: ignore[misc] # decorated prop not supported in mypy def patches(self): @@ -3611,19 +3645,16 @@ def patches(self): # translate patch sha256sums to patch objects by consulting the index if self._patches_assigned(): - for sha256 in self.variants["patches"]._patches_in_order_of_appearance: - index = spack.repo.PATH.patch_index - pkg_cls = spack.repo.PATH.get_pkg_class(self.name) - try: - patch = index.patch_for_package(sha256, pkg_cls) - except spack.error.PatchLookupError as e: - raise spack.error.SpecError( - f"{e}. This usually means the patch was modified or removed. " - "To fix this, either reconcretize or use the original package " - "repository" - ) from e - - self._patches.append(patch) + sha256s = list(self.variants["patches"]._patches_in_order_of_appearance) + pkg_cls = spack.repo.PATH.get_pkg_class(self.name) + try: + self._patches = spack.repo.PATH.get_patches_for_package(sha256s, pkg_cls) + except spack.error.PatchLookupError as e: + raise spack.error.SpecError( + f"{e}. This usually means the patch was modified or removed. " + "To fix this, either reconcretize or use the original package " + "repository" + ) from e return self._patches @@ -3927,7 +3958,7 @@ def _cmp_iter(self): # to do fast equality comparison. See _cmp_fast_eq() above for the # short-circuit logic for hashes. # - # A full traversal involves constructing data structurs, visitor objects, etc., + # A full traversal involves constructing data structures, visitor objects, etc., # and it can be expensive if we have to do it to compare a bunch of tiny # abstract specs. Therefore, there are 3 cases below, which avoid calling # `spack.traverse.traverse_edges()` unless necessary. @@ -4393,7 +4424,7 @@ def _format_edge_attributes(self, dep: DependencySpec, deptypes=True, virtuals=T if deptypes and dep.depflag else "" ) - when_str = f"when='{(dep.when)}'" if dep.when != Spec() else "" + when_str = f"when='{(dep.when)}'" if dep.when != EMPTY_SPEC else "" virtuals_str = f"virtuals={','.join(dep.virtuals)}" if virtuals and dep.virtuals else "" attrs = " ".join(s for s in (when_str, deptypes_str, virtuals_str) if s) @@ -4437,7 +4468,7 @@ def format_edge(edge: DependencySpec, sigil: str, dep_spec: Optional[Spec] = Non edge_attributes = ( self._format_edge_attributes(edge, deptypes=deptypes, virtuals=False) - if edge.depflag or edge.when != Spec() + if edge.depflag or edge.when != EMPTY_SPEC else "" ) virtuals = f"{','.join(edge.virtuals)}=" if edge.virtuals else "" @@ -4508,6 +4539,10 @@ def _short_spec(self, color: Optional[bool] = False) -> str: @property def compilers(self): + if self.original_spec_format() < 5: + # These specs don't have compilers as dependencies, return the compiler node attribute + return f" %{self.compiler}" + # TODO: get rid of the space here and make formatting smarter return " " + self._format_dependencies( "{name}{@version}", @@ -5044,9 +5079,18 @@ def __hash__(self): self._dunder_hash = self.dag_hash_bit_prefix(64) return self._dunder_hash - # This is the normal hash for lazy_lexicographic_ordering. It's - # slow for large specs because it traverses the whole spec graph, - # so we hope it only runs on abstract specs, which are small. + if not self._dependencies: + return hash( + ( + self.name, + self.namespace, + self.versions, + (self.variants if self.variants.dict else None), + self.architecture, + self.abstract_hash, + ) + ) + return hash(lang.tuplify(self._cmp_iter)) def __getstate__(self): @@ -5079,8 +5123,8 @@ def __setstate__(self, state): self._package = None # Reconstruct variants and compiler_flags - self.variants = VariantMap(self) - self.compiler_flags = FlagMap(self) + self.variants = VariantMap() + self.compiler_flags = FlagMap() if variants_data is not None: self.variants.dict = variants_data if compiler_flags_data is not None: @@ -5116,10 +5160,6 @@ class VariantMap(lang.HashableMap[str, vt.VariantValue]): """Map containing variant instances. New values can be added only if the key is not already present.""" - def __init__(self, spec: Spec): - super().__init__() - self.spec = spec - def __setitem__(self, name, vspec): # Raise a TypeError if vspec is not of the right type if not isinstance(vspec, vt.VariantValue): @@ -5136,7 +5176,7 @@ def __setitem__(self, name, vspec): # Raise an error if name and vspec.name don't match if name != vspec.name: raise KeyError( - f'Inconsistent key "{name}", must be "{vspec.name}" to ' "match VariantSpec" + f'Inconsistent key "{name}", must be "{vspec.name}" to match VariantSpec' ) # Set the item @@ -5161,90 +5201,8 @@ def partition_variants(self): prop = [x.name for x in prop] return non_prop, prop - def satisfies(self, other: "VariantMap") -> bool: - if self.spec.concrete: - return self._satisfies_when_self_concrete(other) - return self._satisfies_when_self_abstract(other) - - def _satisfies_when_self_concrete(self, other: "VariantMap") -> bool: - non_propagating, propagating = other.partition_variants() - result = all( - name in self and self[name].satisfies(other[name]) for name in non_propagating - ) - if not propagating: - return result - - for node in self.spec.traverse(): - if not all( - node.variants[name].satisfies(other[name]) - for name in propagating - if name in node.variants - ): - return False - return result - - def _satisfies_when_self_abstract(self, other: "VariantMap") -> bool: - other_non_propagating, other_propagating = other.partition_variants() - self_non_propagating, self_propagating = self.partition_variants() - - # First check variants without propagation set - result = all( - name in self_non_propagating - and (self[name].propagate or self[name].satisfies(other[name])) - for name in other_non_propagating - ) - if result is False or (not other_propagating and not self_propagating): - return result - - # Check that self doesn't contradict variants propagated by other - if other_propagating: - for node in self.spec.traverse(): - if not all( - node.variants[name].satisfies(other[name]) - for name in other_propagating - if name in node.variants - ): - return False - - # Check that other doesn't contradict variants propagated by self - if self_propagating: - for node in other.spec.traverse(): - if not all( - node.variants[name].satisfies(self[name]) - for name in self_propagating - if name in node.variants - ): - return False - - return result - - def intersects(self, other): - return all(self[k].intersects(other[k]) for k in other if k in self) - - def constrain(self, other: "VariantMap") -> bool: - """Add all variants in other that aren't in self to self. Also constrain all multi-valued - variants that are already present. Return True iff self changed""" - if other.spec is not None and other.spec._concrete: - for k in self: - if k not in other: - raise vt.UnsatisfiableVariantSpecError(self[k], "") - - changed = False - for k in other: - if k in self: - if not self[k].intersects(other[k]): - raise vt.UnsatisfiableVariantSpecError(self[k], other[k]) - # If they are compatible merge them - changed |= self[k].constrain(other[k]) - else: - # If it is not present copy it straight away - self[k] = other[k].copy() - changed = True - - return changed - def copy(self) -> "VariantMap": - clone = VariantMap(self.spec) + clone = VariantMap() for name, variant in self.items(): clone[name] = variant.copy() return clone @@ -5278,6 +5236,47 @@ def partition_keys(self) -> Tuple[List[str], List[str]]: return bool_keys, kv_keys +class SpecBuildInterface(lang.ObjectWrapper, Spec): + # home is available in the base Package so no default is needed + home = ForwardQueryToPackage("home", default_handler=None) + headers = ForwardQueryToPackage("headers", default_handler=_headers_default_handler) + libs = ForwardQueryToPackage("libs", default_handler=_libs_default_handler) + command = ForwardQueryToPackage("command", default_handler=None, _indirect=True) + + def __init__( + self, + spec: "Spec", + name: str, + query_parameters: List[str], + _parent: "Spec", + is_virtual: bool, + ): + lang.ObjectWrapper.__init__(self, spec) + # Adding new attributes goes after ObjectWrapper.__init__ call since the ObjectWrapper + # resets __dict__ to behave like the passed object + original_spec = getattr(spec, "wrapped_obj", spec) + self.wrapped_obj = original_spec + self.token = original_spec, name, query_parameters, _parent, is_virtual + self.last_query = QueryState( + name=name, extra_parameters=query_parameters, isvirtual=is_virtual + ) + + # TODO: this ad-hoc logic makes `spec["python"].command` return + # `spec["python-venv"].command` and should be removed when `python` is a virtual. + self.indirect_spec = None + if spec.name == "python": + python_venvs = _parent.dependencies("python-venv") + if not python_venvs: + return + self.indirect_spec = python_venvs[0] + + def __reduce__(self): + return SpecBuildInterface, self.token + + def copy(self, *args, **kwargs): + return self.wrapped_obj.copy(*args, **kwargs) + + def substitute_abstract_variants(spec: Spec): """Uses the information in ``spec.package`` to turn any variant that needs it into a SingleValuedVariant or BoolValuedVariant. @@ -5390,6 +5389,8 @@ def reconstruct_virtuals_on_edges(spec: Spec) -> None: class SpecfileReaderBase: + SPEC_VERSION: int + @classmethod def from_node_dict(cls, node): spec = Spec() @@ -5531,6 +5532,10 @@ def _load(cls, data): return hash_dict[root_spec_hash]["node_spec"] + @classmethod + def extract_build_spec_info_from_node_dict(cls, node, hash_type=ht.dag_hash.name): + raise NotImplementedError("Subclasses must implement this method.") + @classmethod def read_specfile_dep_specs(cls, deps, hash_type=ht.dag_hash.name): raise NotImplementedError("Subclasses must implement this method.") @@ -6004,3 +6009,39 @@ class InvalidEdgeError(spack.error.SpecError): class SpecMutationError(spack.error.SpecError): """Raised when a mutation is attempted with invalid attributes.""" + + +class _ImmutableSpec(Spec): + """An immutable Spec that prevents a class of accidental mutations.""" + + _mutable: bool + + def __init__(self, spec_like: Optional[str] = None) -> None: + object.__setattr__(self, "_mutable", True) + super().__init__(spec_like) + object.__delattr__(self, "_mutable") + + def __setstate__(self, state) -> None: + object.__setattr__(self, "_mutable", True) + super().__setstate__(state) + object.__delattr__(self, "_mutable") + + def constrain(self, *args, **kwargs) -> bool: + assert self._mutable + return super().constrain(*args, **kwargs) + + def add_dependency_edge(self, *args, **kwargs): + assert self._mutable + return super().add_dependency_edge(*args, **kwargs) + + def __setattr__(self, name, value) -> None: + assert self._mutable + super().__setattr__(name, value) + + def __delattr__(self, name) -> None: + assert self._mutable + object.__delattr__(self, name) + + +#: Immutable empty spec, for fast comparisons and reduced memory usage. +EMPTY_SPEC = _ImmutableSpec() diff --git a/lib/spack/spack/spec_filter.py b/lib/spack/spack/spec_filter.py new file mode 100644 index 00000000000000..039df55bb473ea --- /dev/null +++ b/lib/spack/spack/spec_filter.py @@ -0,0 +1,48 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from typing import Callable, List + +import spack.spec + + +class SpecFilter: + """Given a method to produce a list of specs, this class can filter them according to + different criteria. + """ + + def __init__( + self, + factory: Callable[[], List[spack.spec.Spec]], + is_usable: Callable[[spack.spec.Spec], bool], + include: List[str], + exclude: List[str], + ) -> None: + """ + Args: + factory: factory to produce a list of specs + is_usable: predicate that takes a spec in input and returns False if the spec + should not be considered for this filter, True otherwise. + include: if present, a spec must match at least one entry in the list, + to be in the output + exclude: if present, a spec must not match any entry in the list to be in the output + """ + self.factory = factory + self.is_usable = is_usable + self.include = include + self.exclude = exclude + + def is_selected(self, s: spack.spec.Spec) -> bool: + if not self.is_usable(s): + return False + + if self.include and not any(s.satisfies(c) for c in self.include): + return False + + if self.exclude and any(s.satisfies(c) for c in self.exclude): + return False + + return True + + def selected_specs(self) -> List[spack.spec.Spec]: + return [s for s in self.factory() if self.is_selected(s)] diff --git a/lib/spack/spack/spec_parser.py b/lib/spack/spack/spec_parser.py index 1d5d66691b46c8..67f5e0e2516108 100644 --- a/lib/spack/spack/spec_parser.py +++ b/lib/spack/spack/spec_parser.py @@ -56,13 +56,13 @@ specs to avoid ambiguity. Both are provided because ``~`` can cause shell expansion when it is the first character in an id typed on the command line. """ + import json import pathlib import re import sys from typing import TYPE_CHECKING, Dict, Iterator, List, Optional, Tuple, Union -import spack.config import spack.deptypes import spack.error import spack.version @@ -290,19 +290,12 @@ def parse_virtual_assignment(context: TokenContext) -> Tuple[str]: class SpecParser: """Parse text into specs""" - __slots__ = "literal_str", "ctx", "toolchains", "parsed_toolchains" + __slots__ = "literal_str", "ctx" def __init__(self, literal_str: str): self.literal_str = literal_str self.ctx = TokenContext(parseable_tokens(literal_str)) - # TODO: Move toolchains out of the parser, and expand them as a separate step - self.toolchains = {} - configuration = getattr(spack.config, "CONFIG", None) - if configuration is not None: - self.toolchains = configuration.get_config("toolchains") - self.parsed_toolchains: Dict[str, "spack.spec.Spec"] = {} - def tokens(self) -> List[Token]: """Return the entire list of token from the initial text. White spaces are filtered out. @@ -339,71 +332,39 @@ def add_dependency(dep, **edge_properties): current_spec = root_spec while True: if self.ctx.accept(SpecTokens.START_EDGE_PROPERTIES): - is_direct = self.ctx.current_token.value[0] == "%" + has_edge_attrs = True + elif self.ctx.accept(SpecTokens.DEPENDENCY): + has_edge_attrs = False + else: + break - propagation = PropagationPolicy.NONE - if is_direct and self.ctx.current_token.value.startswith("%%"): - propagation = PropagationPolicy.PREFERENCE + is_direct = self.ctx.current_token.value[0] == "%" + propagation = PropagationPolicy.NONE + if is_direct and self.ctx.current_token.value.startswith("%%"): + propagation = PropagationPolicy.PREFERENCE + if has_edge_attrs: edge_properties = EdgeAttributeParser(self.ctx, self.literal_str).parse() edge_properties.setdefault("virtuals", ()) - edge_properties["direct"] = is_direct edge_properties.setdefault("depflag", 0) - edge_properties["propagation"] = propagation - - dependency = self._parse_node(root_spec) - - if is_direct: - target_spec = current_spec - if dependency.name in LEGACY_COMPILER_TO_BUILTIN: - dependency.name = LEGACY_COMPILER_TO_BUILTIN[dependency.name] - else: - current_spec = dependency - target_spec = root_spec - - add_dependency(dependency, **edge_properties) - - elif self.ctx.accept(SpecTokens.DEPENDENCY): - is_direct = self.ctx.current_token.value[0] == "%" - propagation = PropagationPolicy.NONE - - if is_direct and self.ctx.current_token.value.startswith("%%"): - propagation = PropagationPolicy.PREFERENCE - + else: virtuals = parse_virtual_assignment(self.ctx) + edge_properties = {"virtuals": virtuals, "depflag": 0} - # if no virtual assignment, check for a toolchain - look ahead to find the - # toolchain and substitute it - if not virtuals and is_direct and self.ctx.next_token.value in self.toolchains: - assert self.ctx.accept(SpecTokens.UNQUALIFIED_PACKAGE_NAME) - try: - self._apply_toolchain( - current_spec, self.ctx.current_token.value, propagation=propagation - ) - except spack.error.SpecError as e: - raise SpecParsingError(str(e), self.ctx.current_token, self.literal_str) - continue - - edge_properties = { - "direct": is_direct, - "virtuals": virtuals, - "depflag": 0, - "propagation": propagation, - } - dependency = self._parse_node(root_spec) - - if is_direct: - target_spec = current_spec - if dependency.name in LEGACY_COMPILER_TO_BUILTIN: - dependency.name = LEGACY_COMPILER_TO_BUILTIN[dependency.name] - else: - current_spec = dependency - target_spec = root_spec - - add_dependency(dependency, **edge_properties) + edge_properties["direct"] = is_direct + edge_properties["propagation"] = propagation + dependency = self._parse_node(root_spec) + + if is_direct: + target_spec = current_spec + if dependency.name in LEGACY_COMPILER_TO_BUILTIN: + dependency.name = LEGACY_COMPILER_TO_BUILTIN[dependency.name] else: - break + current_spec = dependency + target_spec = root_spec + + add_dependency(dependency, **edge_properties) return root_spec @@ -419,48 +380,6 @@ def _parse_node(self, root_spec: "spack.spec.Spec", root: bool = True): raise spack.error.SpecError(str(root_spec), "^" + str(dependency)) return dependency - def _apply_toolchain( - self, spec: "spack.spec.Spec", name: str, *, propagation: PropagationPolicy - ) -> None: - if name not in self.parsed_toolchains: - toolchain = self._parse_toolchain(name) - self.parsed_toolchains[name] = toolchain - - propagation_arg = None if propagation != PropagationPolicy.PREFERENCE else propagation - # Here we need to copy because we want "foo %toolc ^bar %toolc" to generate different - # objects for the toolc attached to foo and bar, since the solver depends on that to - # generate facts - toolchain = self.parsed_toolchains[name].copy(propagation=propagation_arg) - spec.constrain(toolchain) - - def _parse_toolchain(self, name: str) -> "spack.spec.Spec": - toolchain_config = self.toolchains[name] - if isinstance(toolchain_config, str): - toolchain = parse_one_or_raise(toolchain_config) - self._ensure_all_direct_edges(toolchain) - else: - from spack.spec import Spec - - toolchain = Spec() - for entry in toolchain_config: - toolchain_part = parse_one_or_raise(entry["spec"]) - when = entry.get("when", "") - self._ensure_all_direct_edges(toolchain_part) - - # Conditions are applied to every edge in the constraint - for edge in toolchain_part.traverse_edges(): - edge.when.constrain(when) - toolchain.constrain(toolchain_part) - return toolchain - - def _ensure_all_direct_edges(self, constraint: "spack.spec.Spec") -> None: - for edge in constraint.traverse_edges(root=False): - if not edge.direct: - raise spack.error.SpecError( - f"cannot use '^' in toolchain definitions, and the current " - f"toolchain contains '{edge.format()}'" - ) - def all_specs(self) -> List["spack.spec.Spec"]: """Return all the specs that remain to be parsed""" return list(iter(self.next_spec, None)) @@ -655,26 +574,36 @@ def parse(self): return attributes -def parse(text: str) -> List["spack.spec.Spec"]: - """Parse text into a list of strings +def parse(text: str, *, toolchains: Optional[Dict] = None) -> List["spack.spec.Spec"]: + """Parse text into a list of specs Args: - text (str): text to be parsed + text: text to be parsed + toolchains: optional toolchain definitions to expand after parsing Return: List of specs """ - return SpecParser(text).all_specs() + specs = SpecParser(text).all_specs() + if toolchains: + cache: Dict[str, "spack.spec.Spec"] = {} + for spec in specs: + expand_toolchains(spec, toolchains, _cache=cache) + return specs def parse_one_or_raise( - text: str, initial_spec: Optional["spack.spec.Spec"] = None + text: str, + initial_spec: Optional["spack.spec.Spec"] = None, + *, + toolchains: Optional[Dict] = None, ) -> "spack.spec.Spec": """Parse exactly one spec from text and return it, or raise Args: - text (str): text to be parsed + text: text to be parsed initial_spec: buffer where to parse the spec. If None a new one will be created. + toolchains: optional toolchain definitions to expand after parsing """ parser = SpecParser(text) result = parser.next_spec(initial_spec) @@ -689,16 +618,97 @@ def parse_one_or_raise( if result is None: raise ValueError("expected a single spec, but got none") + if toolchains: + expand_toolchains(result, toolchains) + return result +def _parse_toolchain_config(toolchain_config: Union[str, List[Dict]]) -> "spack.spec.Spec": + """Parse a toolchain config entry (string or list) into a Spec.""" + if isinstance(toolchain_config, str): + toolchain = parse_one_or_raise(toolchain_config) + _ensure_all_direct_edges(toolchain) + else: + from spack.spec import EMPTY_SPEC, Spec + + toolchain = Spec() + for entry in toolchain_config: + toolchain_part = parse_one_or_raise(entry["spec"]) + when = entry.get("when", "") + _ensure_all_direct_edges(toolchain_part) + + if when: + when_spec = Spec(when) + for edge in toolchain_part.traverse_edges(): + if edge.when is EMPTY_SPEC: + edge.when = when_spec.copy() + else: + edge.when.constrain(when_spec) + toolchain.constrain(toolchain_part) + return toolchain + + +def _ensure_all_direct_edges(constraint: "spack.spec.Spec") -> None: + """Validate that a toolchain spec only has direct (%) edges.""" + for edge in constraint.traverse_edges(root=False): + if not edge.direct: + raise spack.error.SpecError( + f"cannot use '^' in toolchain definitions, and the current " + f"toolchain contains '{edge.format()}'" + ) + + +def expand_toolchains( + spec: "spack.spec.Spec", + toolchains: Dict, + *, + _cache: Optional[Dict[str, "spack.spec.Spec"]] = None, +) -> None: + """Replace toolchain placeholder deps with expanded toolchain constraints. + + Walks every node in the spec DAG. For each node, finds direct dependency + edges whose child name is a key in ``toolchains``. Removes the placeholder + edge, parses the toolchain config, copies with the edge's propagation + policy, and constrains the node. + """ + if _cache is None: + _cache = {} + + for node in list(spec.traverse()): + for edge in list(node.edges_to_dependencies()): + if not edge.direct: + continue + name = edge.spec.name + if name not in toolchains: + continue + + # Remove the placeholder edge (both directions) + node._dependencies[name].remove(edge) + if not node._dependencies[name]: + del node._dependencies[name] + edge.spec._dependents[node.name].remove(edge) + if not edge.spec._dependents[node.name]: + del edge.spec._dependents[node.name] + + # Parse and cache toolchain + if name not in _cache: + _cache[name] = _parse_toolchain_config(toolchains[name]) + + propagation = edge.propagation + propagation_arg = None if propagation != PropagationPolicy.PREFERENCE else propagation + # Copy so each usage gets a distinct object (solver depends on this) + toolchain = _cache[name].copy(propagation=propagation_arg) + node.constrain(toolchain) + + class SpecParsingError(spack.error.SpecSyntaxError): """Error when parsing tokens""" def __init__(self, message, token, text): message += f"\n{text}" if token: - underline = f"\n{' '*token.start}{'^'*(token.end - token.start)}" + underline = f"\n{' ' * token.start}{'^' * (token.end - token.start)}" message += color.colorize(f"@*r{{{underline}}}") super().__init__(message) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 94a6d9053be34f..c98ee2f495a015 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -67,7 +67,7 @@ def compute_stage_name(spec): else: spec_stage_structure += "{name}-{version}" # TODO (psakiev, scheibelp) Technically a user could still reintroduce a hash via - # config:stage_name. This is a fix for how to handle staging an abstact spec (see #51305) + # config:stage_name. This is a fix for how to handle staging an abstract spec (see #51305) stage_name_structure = spack.config.get("config:stage_name", default=spec_stage_structure) return spec.format_path(format_string=stage_name_structure) @@ -396,7 +396,7 @@ class Stage(AbstractStage): When used as a context manager, the stage is automatically destroyed if no exception is raised by the context. If an - excpetion is raised, the stage is left in the filesystem and NOT + exception is raised, the stage is left in the filesystem and NOT destroyed, for potential reuse later. You can also use the stage's create/destroy functions manually, @@ -974,13 +974,16 @@ def __init__(self, name, dev_path, reference_link): self._source_path = dev_path # The path of a link that will point to this stage - if os.path.isabs(reference_link): - link_path = reference_link + if reference_link: + if os.path.isabs(reference_link): + link_path = reference_link + else: + link_path = os.path.join(self._source_path, reference_link) + if not os.path.isdir(os.path.dirname(link_path)): + raise StageError(f"The directory containing {link_path} must exist") + self.reference_link = link_path else: - link_path = os.path.join(self._source_path, reference_link) - if not os.path.isdir(os.path.dirname(link_path)): - raise StageError(f"The directory containing {link_path} must exist") - self.reference_link = link_path + self.reference_link = None @property def source_path(self): @@ -1007,10 +1010,11 @@ def expanded(self): def create(self): super().create() - try: - symlink(self.path, self.reference_link) - except (AlreadyExistsError, FileExistsError): - pass + if self.reference_link: + try: + symlink(self.path, self.reference_link) + except (AlreadyExistsError, FileExistsError): + pass def destroy(self): # Destroy all files, but do not follow symlinks @@ -1018,10 +1022,11 @@ def destroy(self): shutil.rmtree(self.path) except FileNotFoundError: pass - try: - os.remove(self.reference_link) - except FileNotFoundError: - pass + if self.reference_link: + try: + os.remove(self.reference_link) + except FileNotFoundError: + pass self.created = False def restage(self): @@ -1294,7 +1299,7 @@ def get_checksums_for_versions( else: version_hashes[version] = result - with spack.util.parallel.make_concurrent_executor(concurrency, require_fork=False) as executor: + with spack.util.parallel.make_concurrent_executor(concurrency) as executor: results = [ (version, executor.submit(_fetch_and_checksum, url, fetch_options, keep_stage)) for url, version in search_arguments diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index 4068398586a790..e8f07e70f19bb4 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -15,21 +15,28 @@ debugging easier. """ + import contextlib +import filecmp import os import pathlib import re +import secrets +import shutil +import sys import uuid -from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union, cast import spack.config import spack.database import spack.directory_layout import spack.error import spack.llnl.util.lang +import spack.package_prefs import spack.paths import spack.spec import spack.util.path +from spack.llnl.util import filesystem as fs from spack.llnl.util import tty #: default installation root, relative to the Spack install path @@ -185,10 +192,66 @@ def __init__( self.root, default_timeout=lock_cfg.package_timeout ) + def has_padding(self) -> bool: + """Returns True if the store layout includes path padding.""" + return self.root != self.unpadded_root + def reindex(self) -> None: """Convenience function to reindex the store DB with its own layout.""" return self.db.reindex() + def install_sbang(self) -> None: + """Install the sbang script in this store's bin directory. + + sbang is a short shell script that Spack prepends to scripts with shebangs that are too + long for the OS. It must live in the store so its path is short enough to fit on a + shebang line. + """ + + if sys.platform == "win32": + return + + import grp # unix only, hence the import here + + sbang_path = os.path.join(self.unpadded_root, "bin", "sbang") + try: + if filecmp.cmp(sbang_path, spack.paths.sbang_script): + return # installed and up to date + except FileNotFoundError: + pass + + bin_dir = os.path.dirname(sbang_path) + os.makedirs(bin_dir, exist_ok=True) + + all_spec = spack.spec.Spec("all") + group_name = spack.package_prefs.get_package_group(all_spec) + config_mode = spack.package_prefs.get_package_dir_permissions(all_spec) + gid = grp.getgrnam(group_name).gr_gid if group_name else -1 + + if group_name: + os.chmod(bin_dir, config_mode) + os.chown(bin_dir, -1, gid) + else: + fs.set_install_permissions(bin_dir) + + sbang_tmp_path = os.path.join(bin_dir, f".sbang.{secrets.token_hex(8)}.tmp") + # Open a randomized temporary file with O_EXCL to error on races. Outside the try-except + # to ensure we don't delete a file created by another process in the except block. + sbang_tmp_file = open(sbang_tmp_path, "xb") + try: + with open(spack.paths.sbang_script, "rb") as src, sbang_tmp_file as dst: + shutil.copyfileobj(src, dst) + os.fchmod(dst.fileno(), config_mode | 0o111) # ensure executable + if group_name: + os.fchown(dst.fileno(), -1, gid) + os.rename(sbang_tmp_path, sbang_path) + except BaseException: + try: + os.unlink(sbang_tmp_path) + except OSError: + pass + raise + def __reduce__(self): return Store, ( self.root, @@ -233,7 +296,7 @@ def _create_global() -> Store: #: Singleton store instance -STORE: Store = spack.llnl.util.lang.Singleton(_create_global) # type: ignore +STORE = cast(Store, spack.llnl.util.lang.Singleton(_create_global)) def reinitialize(): @@ -243,7 +306,7 @@ def reinitialize(): global STORE token = STORE - STORE = spack.llnl.util.lang.Singleton(_create_global) + STORE = cast(Store, spack.llnl.util.lang.Singleton(_create_global)) return token diff --git a/lib/spack/spack/subprocess_context.py b/lib/spack/spack/subprocess_context.py index c48b642d7763e9..0d34234ee929b3 100644 --- a/lib/spack/spack/subprocess_context.py +++ b/lib/spack/spack/subprocess_context.py @@ -11,6 +11,7 @@ modifications to global state in memory that must be replicated in the child process. """ + import importlib import io import multiprocessing @@ -71,7 +72,7 @@ def __init__( ctx: Optional[multiprocessing.context.BaseContext] = None, ): ctx = ctx or multiprocessing.get_context() - self.global_state = GlobalStateMarshaler(ctx=ctx) + self.global_state = GlobalStateMarshaler(ctx=ctx, serialize_env=True) self.pkg = pkg if ctx.get_start_method() == "fork" else serialize(pkg) self.spack_working_dir = spack.paths.spack_working_dir @@ -89,23 +90,38 @@ class GlobalStateMarshaler: """ def __init__( - self, *, ctx: Optional[Optional[multiprocessing.context.BaseContext]] = None + self, + *, + ctx: Optional[Optional[multiprocessing.context.BaseContext]] = None, + serialize_env: bool = False, ) -> None: ctx = ctx or multiprocessing.get_context() self.is_forked = ctx.get_start_method() == "fork" if self.is_forked: return - from spack.environment import active_environment - self.config = spack.config.CONFIG.ensure_unwrapped() self.platform = spack.platforms.host self.store = spack.store.STORE self.test_patches = TestPatches.create() - self.env = active_environment() + if serialize_env: + from spack.environment import active_environment + + self.env = active_environment() + else: + self.env = None def restore(self): if self.is_forked: + # Erase singletons that hold open SSL contexts / boto3 clients, since OpenSSL + # and botocore connection pools are not fork-safe. + from spack.oci import opener + from spack.util import web + from spack.util.s3 import s3_client_cache + + web.urlopen._instance = None + opener.urlopen._instance = None + s3_client_cache.clear() return spack.config.CONFIG = self.config spack.repo.enable_repo(spack.repo.RepoPath.from_config(self.config)) diff --git a/lib/spack/spack/tag.py b/lib/spack/spack/tag.py index 0e40759337fdf3..855c24bf938e1f 100644 --- a/lib/spack/spack/tag.py +++ b/lib/spack/spack/tag.py @@ -2,7 +2,8 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Classes and functions to manage package tags""" -from typing import TYPE_CHECKING, Dict, List + +from typing import TYPE_CHECKING, Dict, List, Set import spack.error import spack.util.spack_json as sjson @@ -51,26 +52,28 @@ def merge(self, other: "TagIndex") -> None: else: self.tags[tag] = sorted({*self.tags[tag], *pkgs}) - def update_package(self, pkg_name: str, repo: "spack.repo.Repo") -> None: - """Updates a package in the tag index. + def update_packages(self, pkg_names: Set[str], repo: "spack.repo.Repo") -> None: + """Updates packages in the tag index. Args: - pkg_name: name of the package to be updated + pkg_names: names of the packages to be updated + repo: the repository to get package classes from """ - pkg_cls = repo.get_pkg_class(pkg_name) - - # Remove the package from the list of packages, if present + # Remove the packages from the list of packages, if present for pkg_list in self.tags.values(): - if pkg_name in pkg_list: - pkg_list.remove(pkg_name) - - # Add it again under the appropriate tags - for tag in getattr(pkg_cls, "tags", []): - tag = tag.lower() - if tag not in self.tags: - self.tags[tag] = [pkg_cls.name] - else: - self.tags[tag].append(pkg_cls.name) + if pkg_names.isdisjoint(pkg_list): + continue + pkg_list[:] = [pkg for pkg in pkg_list if pkg not in pkg_names] + + # Add them again under the appropriate tags + for pkg_name in pkg_names: + pkg_cls = repo.get_pkg_class(pkg_name) + for tag in getattr(pkg_cls, "tags", []): + tag = tag.lower() + if tag not in self.tags: + self.tags[tag] = [pkg_cls.name] + else: + self.tags[tag].append(pkg_cls.name) class TagIndexError(spack.error.SpackError): diff --git a/lib/spack/spack/test/audit.py b/lib/spack/spack/test/audit.py index 5c47388674d834..12c14784903e42 100644 --- a/lib/spack/spack/test/audit.py +++ b/lib/spack/spack/test/audit.py @@ -28,12 +28,12 @@ (["invalid-selfhosted-gitlab-patch-url"], ["PKG-DIRECTIVES", "PKG-PROPERTIES"]), # This package has a stand-alone test method in build-time callbacks (["fail-test-audit"], ["PKG-PROPERTIES"]), - # This package implements and uses several deprecated stand-alone test methods - (["fail-test-audit-deprecated"], ["PKG-DEPRECATED-ATTRIBUTES"]), # This package has stand-alone test methods without non-trivial docstrings (["fail-test-audit-docstring"], ["PKG-PROPERTIES"]), # This package has a stand-alone test method without an implementation (["fail-test-audit-impl"], ["PKG-PROPERTIES"]), + # This package has maintainers with placeholders + (["invalid-maintainer"], ["PKG-DIRECTIVES"]), # This package has no issues (["mpileaks"], None), # This package has a conflict with a trigger which cannot constrain the constraint diff --git a/lib/spack/spack/test/binary_distribution.py b/lib/spack/spack/test/binary_distribution.py index d677a53c35bf08..e35631e1c35655 100644 --- a/lib/spack/spack/test/binary_distribution.py +++ b/lib/spack/spack/test/binary_distribution.py @@ -13,6 +13,7 @@ import urllib.error import urllib.request import urllib.response +import warnings from pathlib import Path, PurePath from typing import Any, Callable, Dict, NamedTuple, Optional @@ -273,8 +274,11 @@ def test_use_bin_index(monkeypatch, tmp_path: pathlib.Path, mutable_config): """Check use of binary cache index: perform an operation that instantiates it, and a second operation that reconstructs it. """ + index_cache_root = str(tmp_path / "index_cache") monkeypatch.setattr( - spack.binary_distribution, "BINARY_INDEX", spack.binary_distribution.BinaryCacheIndex() + spack.binary_distribution, + "BINARY_INDEX", + spack.binary_distribution.BinaryIndexCache(index_cache_root), ) # Create a mirror, configure us to point at it, install a spec, and @@ -289,7 +293,9 @@ def test_use_bin_index(monkeypatch, tmp_path: pathlib.Path, mutable_config): # Now the test buildcache_cmd("list", "-al") - spack.binary_distribution.BINARY_INDEX = spack.binary_distribution.BinaryCacheIndex() + spack.binary_distribution.BINARY_INDEX = spack.binary_distribution.BinaryIndexCache( + index_cache_root + ) cache_list = buildcache_cmd("list", "-al") assert "libdwarf" in cache_list @@ -301,8 +307,11 @@ def test_use_bin_index_active_env_with_view( """Check use of binary cache index: perform an operation that instantiates it, and a second operation that reconstructs it. """ + index_cache_root = str(tmp_path / "index_cache") monkeypatch.setattr( - spack.binary_distribution, "BINARY_INDEX", spack.binary_distribution.BinaryCacheIndex() + spack.binary_distribution, + "BINARY_INDEX", + spack.binary_distribution.BinaryIndexCache(index_cache_root), ) # Create a mirror, configure us to point at it, install a spec, and @@ -321,7 +330,9 @@ def test_use_bin_index_active_env_with_view( # Now the test buildcache_cmd("list", "-al") - spack.binary_distribution.BINARY_INDEX = spack.binary_distribution.BinaryCacheIndex() + spack.binary_distribution.BINARY_INDEX = spack.binary_distribution.BinaryIndexCache( + index_cache_root + ) cache_list = buildcache_cmd("list", "-al") assert "libdwarf" in cache_list @@ -333,8 +344,11 @@ def test_use_bin_index_with_view( """Check use of binary cache index: perform an operation that instantiates it, and a second operation that reconstructs it. """ + index_cache_root = str(tmp_path / "index_cache") monkeypatch.setattr( - spack.binary_distribution, "BINARY_INDEX", spack.binary_distribution.BinaryCacheIndex() + spack.binary_distribution, + "BINARY_INDEX", + spack.binary_distribution.BinaryIndexCache(index_cache_root), ) # Create a mirror, configure us to point at it, install a spec, and @@ -354,7 +368,9 @@ def test_use_bin_index_with_view( # Now the test buildcache_cmd("list", "-al") - spack.binary_distribution.BINARY_INDEX = spack.binary_distribution.BinaryCacheIndex() + spack.binary_distribution.BINARY_INDEX = spack.binary_distribution.BinaryIndexCache( + index_cache_root + ) cache_list = buildcache_cmd("list", "-al") assert "libdwarf" in cache_list @@ -557,12 +573,16 @@ def response_304(request: urllib.request.Request): if url == f"https://www.example.com/build_cache/{INDEX_JSON_FILE}": assert request.get_header("If-none-match") == '"112a8bbc1b3f7f185621c1ee335f0502"' raise urllib.error.HTTPError( - url, 304, "Not Modified", hdrs={}, fp=None # type: ignore[arg-type] + url, + 304, + "Not Modified", + hdrs={}, # type: ignore[arg-type] + fp=None, # type: ignore[arg-type] ) assert False, "Should not fetch {}".format(url) - fetcher = spack.binary_distribution.EtagIndexFetcherV2( - url="https://www.example.com", + fetcher = spack.binary_distribution.EtagIndexHandlerV2( + spack.binary_distribution.MirrorMetadata("https://www.example.com", 2), etag="112a8bbc1b3f7f185621c1ee335f0502", urlopen=response_304, ) @@ -586,8 +606,8 @@ def response_200(request: urllib.request.Request): ) assert False, "Should not fetch {}".format(url) - fetcher = spack.binary_distribution.EtagIndexFetcherV2( - url="https://www.example.com", + fetcher = spack.binary_distribution.EtagIndexHandlerV2( + spack.binary_distribution.MirrorMetadata("https://www.example.com", 2), etag="112a8bbc1b3f7f185621c1ee335f0502", urlopen=response_200, ) @@ -611,8 +631,8 @@ def response_404(request: urllib.request.Request): fp=None, ) - fetcher = spack.binary_distribution.EtagIndexFetcherV2( - url="https://www.example.com", + fetcher = spack.binary_distribution.EtagIndexHandlerV2( + spack.binary_distribution.MirrorMetadata("https://www.example.com", 2), etag="112a8bbc1b3f7f185621c1ee335f0502", urlopen=response_404, ) @@ -645,8 +665,10 @@ def urlopen(request: urllib.request.Request): assert False, "Unexpected request {}".format(url) - fetcher = spack.binary_distribution.DefaultIndexFetcherV2( - url="https://www.example.com", local_hash="outdated", urlopen=urlopen + fetcher = spack.binary_distribution.DefaultIndexHandlerV2( + spack.binary_distribution.MirrorMetadata("https://www.example.com", 2), + local_hash="outdated", + urlopen=urlopen, ) result = fetcher.conditional_fetch() @@ -676,8 +698,10 @@ def urlopen(request: urllib.request.Request): assert False, "Unexpected request {}".format(url) - fetcher = spack.binary_distribution.DefaultIndexFetcherV2( - url="https://www.example.com", local_hash=None, urlopen=urlopen + fetcher = spack.binary_distribution.DefaultIndexHandlerV2( + spack.binary_distribution.MirrorMetadata("https://www.example.com", 2), + local_hash=None, + urlopen=urlopen, ) result = fetcher.conditional_fetch() @@ -706,8 +730,10 @@ def urlopen(request: urllib.request.Request): # No request to index.json should be made. assert False, "Unexpected request {}".format(url) - fetcher = spack.binary_distribution.DefaultIndexFetcherV2( - url="https://www.example.com", local_hash=index_json_hash, urlopen=urlopen + fetcher = spack.binary_distribution.DefaultIndexHandlerV2( + spack.binary_distribution.MirrorMetadata("https://www.example.com", 2), + local_hash=index_json_hash, + urlopen=urlopen, ) assert fetcher.conditional_fetch().fresh @@ -726,8 +752,10 @@ def urlopen(request: urllib.request.Request): code=200, ) - fetcher = spack.binary_distribution.DefaultIndexFetcherV2( - url="https://www.example.com", local_hash=index_json_hash, urlopen=urlopen + fetcher = spack.binary_distribution.DefaultIndexHandlerV2( + spack.binary_distribution.MirrorMetadata("https://www.example.com", 2), + local_hash=index_json_hash, + urlopen=urlopen, ) assert fetcher.get_remote_hash() is None @@ -759,8 +787,10 @@ def urlopen(request: urllib.request.Request): assert False, "Unexpected fetch {}".format(url) - fetcher = spack.binary_distribution.DefaultIndexFetcherV2( - url="https://www.example.com", local_hash="invalid", urlopen=urlopen + fetcher = spack.binary_distribution.DefaultIndexHandlerV2( + spack.binary_distribution.MirrorMetadata("https://www.example.com", 2), + local_hash="invalid", + urlopen=urlopen, ) with pytest.raises(spack.binary_distribution.FetchIndexError, match="Could not fetch index"): @@ -1240,11 +1270,15 @@ def response_304(request: urllib.request.Request): if url.endswith(INDEX_MANIFEST_FILE): assert request.get_header("If-none-match") == '"112a8bbc1b3f7f185621c1ee335f0502"' raise urllib.error.HTTPError( - url, 304, "Not Modified", hdrs={}, fp=None # type: ignore[arg-type] + url, + 304, + "Not Modified", + hdrs={}, # type: ignore[arg-type] + fp=None, # type: ignore[arg-type] ) assert False, "Unexpected request {}".format(url) - fetcher = spack.binary_distribution.EtagIndexFetcher( + fetcher = spack.binary_distribution.EtagIndexHandler( spack.binary_distribution.MirrorMetadata( "https://www.example.com", spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION ), @@ -1271,7 +1305,7 @@ def response_200(request: urllib.request.Request): ) assert False, "Unexpected request {}".format(url) - fetcher = spack.binary_distribution.EtagIndexFetcher( + fetcher = spack.binary_distribution.EtagIndexHandler( spack.binary_distribution.MirrorMetadata( "https://www.example.com", spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION ), @@ -1299,7 +1333,7 @@ def response_404(request: urllib.request.Request): fp=None, ) - fetcher = spack.binary_distribution.EtagIndexFetcher( + fetcher = spack.binary_distribution.EtagIndexHandler( spack.binary_distribution.MirrorMetadata( "https://www.example.com", spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION ), @@ -1325,7 +1359,7 @@ def urlopen(request: urllib.request.Request): assert False, "Unexpected request {}".format(url) - fetcher = spack.binary_distribution.DefaultIndexFetcher( + fetcher = spack.binary_distribution.DefaultIndexHandler( spack.binary_distribution.MirrorMetadata( "https://www.example.com", spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION ), @@ -1354,7 +1388,7 @@ def urlopen(request: urllib.request.Request): fp=None, ) - fetcher = spack.binary_distribution.DefaultIndexFetcher( + fetcher = spack.binary_distribution.DefaultIndexHandler( spack.binary_distribution.MirrorMetadata( "https://www.example.com", spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION ), @@ -1381,7 +1415,7 @@ def urlopen(request: urllib.request.Request): # No other request should be made. assert False, "Unexpected request {}".format(url) - fetcher = spack.binary_distribution.DefaultIndexFetcher( + fetcher = spack.binary_distribution.DefaultIndexHandler( spack.binary_distribution.MirrorMetadata( "https://www.example.com", spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION ), @@ -1465,3 +1499,28 @@ def test_mirror_metadata_with_view(): with pytest.raises(spack.url_buildcache.MirrorMetadataError, match="Malformed string"): spack.binary_distribution.MirrorMetadata.from_string("https://dummy.io/__v3%asdf__@aview") + + +def test_update_does_not_warn_on_mirror_with_no_index(monkeypatch, tmp_path, mutable_config): + """Tests that BinaryIndexCache.update() does NOT warn when a mirror has no index but records + that information for later use. + """ + mirror_url = url_util.path_to_file_url(str(tmp_path / "mirror_dir")) + mirror_url2 = url_util.path_to_file_url(str(tmp_path / "mirror_dir2")) + mutable_config.set("mirrors", {"test1": mirror_url, "test2": mirror_url2}) + + def no_index(*args, **kwargs): + raise spack.binary_distribution.BuildcacheIndexNotExists("no index") + + binary_index = spack.binary_distribution.BinaryIndexCache(str(tmp_path / "index_cache")) + monkeypatch.setattr(binary_index, "_fetch_and_cache_index", no_index) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + binary_index.update() + + concretization_warnings = [ + w for w in caught if "cannot be used in concretization" in str(w.message) + ] + assert not concretization_warnings, "update() must not warn about concretization" + assert binary_index.mirrors_without_index == {mirror_url, mirror_url2} diff --git a/lib/spack/spack/test/bootstrap.py b/lib/spack/spack/test/bootstrap.py index 2a9744a397e0dd..9cdcf5999f365a 100644 --- a/lib/spack/spack/test/bootstrap.py +++ b/lib/spack/spack/test/bootstrap.py @@ -10,10 +10,12 @@ import spack.bootstrap.clingo import spack.bootstrap.config import spack.bootstrap.core +import spack.bootstrap.status import spack.compilers.config import spack.config import spack.environment import spack.store +import spack.util.executable import spack.util.path from .conftest import _true @@ -179,9 +181,7 @@ def test_bootstrap_custom_store_in_environment(mutable_config, tmp_path: pathlib config: install_tree: root: {0} -""".format( - install_root - ) +""".format(install_root) ) with spack.environment.Environment(str(tmp_path)): assert spack.environment.active_environment() @@ -206,8 +206,6 @@ def test_nested_use_of_context_manager(mutable_config): def test_status_function_find_files( mutable_config, mock_executable, tmp_path: pathlib.Path, monkeypatch, expected_missing ): - import spack.bootstrap.status - if not expected_missing: mock_executable("foo", "echo Hello WWorld!") @@ -222,6 +220,59 @@ def test_status_function_find_files( assert missing is expected_missing +@pytest.mark.parametrize( + "gpg_in_path,gpg_in_store,expected_missing", + [ + (True, False, False), # gpg exists in PATH + (False, True, False), # gpg exists in bootstrap store + (False, False, True), # gpg is missing + ], +) +def test_gpg_status_check( + mutable_config, + mock_executable, + tmp_path: pathlib.Path, + monkeypatch, + gpg_in_path, + gpg_in_store, + expected_missing, +): + """Test that gpg/gpg2 status is detected whether it's in PATH or in the bootstrap store.""" + # Set up mock PATH with or without gpg + path_dir = tmp_path / "bin" + path_dir.mkdir(exist_ok=True) + monkeypatch.setenv("PATH", str(path_dir)) + + if gpg_in_path: + mock_executable("gpg2", "echo GPG 2.3.4") + + # Mock the bootstrap store function + def mock_executables_in_store(exes, query_spec, query_info=None): + if not gpg_in_store: + return False + + # Simulate found gpg in bootstrap store + if query_info is not None: + query_info["spec"] = "gnupg@2.5.12" + query_info["command"] = spack.util.executable.Executable("gpg") + return True + + monkeypatch.setattr(spack.bootstrap.status, "_executables_in_store", mock_executables_in_store) + + # Call only the buildcache requirements function directly to isolate the test + requirements = spack.bootstrap.status._buildcache_requirements() + + # Find the gpg entry by examining the calls made to set up requirements + # We know the first entry in requirements is the gpg entry because of how + # _buildcache_requirements is structured: + # Make sure we're not out of bounds + assert len(requirements) >= 1, "No gpg requirement found" + + # Check that the gpg requirement matches our expectations + gpg_req = requirements[0] + assert gpg_req[0] is not expected_missing + + @pytest.mark.regression("31042") def test_source_is_disabled(mutable_config): # Get the configuration dictionary of the current bootstrapping source diff --git a/lib/spack/spack/test/build_environment.py b/lib/spack/spack/test/build_environment.py index 1ccd2d4be3e718..4d6b6231686d41 100644 --- a/lib/spack/spack/test/build_environment.py +++ b/lib/spack/spack/test/build_environment.py @@ -20,6 +20,7 @@ import spack.package_base import spack.spec import spack.util.environment +import spack.util.module_cmd import spack.util.spack_yaml as syaml from spack.build_environment import UseMode, _static_to_shared_library, dso_suffix from spack.context import Context @@ -159,7 +160,7 @@ def _set_wrong_cc(x): os.environ["CC"] = "NOT_THIS_PLEASE" os.environ["ANOTHER_VAR"] = "THIS_IS_SET" - monkeypatch.setattr(spack.build_environment, "load_module", _set_wrong_cc) + monkeypatch.setattr(spack.util.module_cmd, "load_module", _set_wrong_cc) s = spack.concretize.concretize_one("cmake %gcc@14") spack.build_environment.setup_package(s.package, dirty=False) @@ -287,6 +288,39 @@ def platform_pathsep(pathlist): assert name not in os.environ +@pytest.mark.not_on_windows("Module files are not supported on Windows") +def test_load_external_modules_error(working_env, monkeypatch): + """Test that load_external_modules raises an exception when a module cannot be loaded""" + + # Create a mock spec object with the minimum attributes needed for the test + class MockSpec: + def __init__(self): + self.external_modules = ["non_existent_module"] + + def __str__(self): + return "mock-external-spec" + + mock_spec = MockSpec() + + # Create a simplified SetupContext-like class that only contains what we need + class MockSetupContext: + def __init__(self, spec): + self.external = [(spec, None)] + + context = MockSetupContext(mock_spec) + + # Mock the load_module function to raise an exception + def mock_load_module(module_name): + # Simulate module load failure + raise spack.util.module_cmd.ModuleLoadError(module_name) + + monkeypatch.setattr(spack.util.module_cmd, "load_module", mock_load_module) + + # Test that load_external_modules raises ModuleLoadError + with pytest.raises(spack.util.module_cmd.ModuleLoadError): + spack.build_environment.load_external_modules(context) + + def test_external_config_env(mock_packages, mutable_config, working_env): cmake_config = { "externals": [ @@ -320,7 +354,7 @@ def test_spack_paths_before_module_paths( def _set_wrong_cc(x): os.environ["PATH"] = module_path + os.pathsep + os.environ["PATH"] - monkeypatch.setattr(spack.build_environment, "load_module", _set_wrong_cc) + monkeypatch.setattr(spack.util.module_cmd, "load_module", _set_wrong_cc) s = spack.concretize.concretize_one("cmake") @@ -607,7 +641,7 @@ def test_effective_deptype_build_environment(default_mock_concretization): # [b ] ^dtbuild1@1.0 # <- direct build dep # [b ] ^dtbuild2@1.0 # <- indirect build-only dep is dropped # [bl ] ^dtlink2@1.0 # <- linkable, and runtime dep of build dep - # [ r ] ^dtrun2@1.0 # <- non-linkable, exectuable runtime dep of build dep + # [ r ] ^dtrun2@1.0 # <- non-linkable, executable runtime dep of build dep # [bl ] ^dtlink1@1.0 # <- direct build dep # [bl ] ^dtlink3@1.0 # <- linkable, and runtime dep of build dep # [b ] ^dtbuild2@1.0 # <- indirect build-only dep is dropped diff --git a/lib/spack/spack/test/builder.py b/lib/spack/spack/test/builder.py index b38eb85505da5c..ed548278a18881 100644 --- a/lib/spack/spack/test/builder.py +++ b/lib/spack/spack/test/builder.py @@ -76,7 +76,9 @@ def builder_test_repository(config): ) @pytest.mark.usefixtures("builder_test_repository", "config") @pytest.mark.disable_clean_stage_check -def test_callbacks_and_installation_procedure(spec_str, expected_values, working_env): +def test_callbacks_and_installation_procedure( + spec_str, expected_values, working_env, temporary_store +): """Test the correct execution of callbacks and installation procedures for packages.""" s = spack.concretize.concretize_one(spec_str) builder = spack.builder.create(s.package) @@ -111,7 +113,7 @@ def test_old_style_compatibility_with_super(spec_str, method_name, expected): @pytest.mark.regression("33928") @pytest.mark.usefixtures("builder_test_repository", "config", "working_env") @pytest.mark.disable_clean_stage_check -def test_build_time_tests_are_executed_from_default_builder(): +def test_build_time_tests_are_executed_from_default_builder(temporary_store): s = spack.concretize.concretize_one("old-style-autotools") builder = spack.builder.create(s.package) builder.pkg.run_tests = True @@ -152,7 +154,9 @@ def test_monkey_patching_test_log_file(): # Windows context manager's __exit__ fails with ValueError ("I/O operation # on closed file"). @pytest.mark.not_on_windows("Does not run on windows") -def test_install_time_test_callback(tmp_path: pathlib.Path, config, mock_packages, mock_stage): +def test_install_time_test_callback( + tmp_path: pathlib.Path, config, mock_packages, mock_stage, temporary_store +): """Confirm able to run stand-alone test as a post-install callback.""" s = spack.concretize.concretize_one("py-test-callback") builder = spack.builder.create(s.package) @@ -215,3 +219,21 @@ class TestBuilder(spack.builder.Builder): assert attributes == ("foo", "bar") long_methods = spack.builder.package_long_methods(TestBuilder) assert long_methods == ("baz", "fee") + + +@pytest.mark.regression("51917") +@pytest.mark.usefixtures("builder_test_repository", "config") +def test_builder_when_inheriting_just_package(working_env): + """Tests that if we inherit a package from another package that has a builder defined, + but we don't need to modify the builder ourselves, we'll get the builder of the base + package class. + """ + base_spec = spack.concretize.concretize_one("callbacks") + derived_spec = spack.concretize.concretize_one("inheritance-only-package") + + base_builder = spack.builder.create(base_spec.package) + derived_builder = spack.builder.create(derived_spec.package) + + # The derived class doesn't redefine a builder, so we should + # get the builder of the base class. + assert type(base_builder) is type(derived_builder) diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index e9aea962085805..03d17e8e088682 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -6,6 +6,7 @@ This test checks that the Spack cc compiler wrapper is parsing arguments correctly. """ + import os import pytest @@ -340,7 +341,7 @@ def test_expected_args(wrapper_environment, wrapper_dir): # Xlinker_parsing # - # -Xlinker ... -Xlinker may have compiler flags inbetween, like -O3 in this + # -Xlinker ... -Xlinker may have compiler flags in between, like -O3 in this # example. Also check that a trailing -Xlinker (which is a compiler error) is not # dropped or given an empty argument. check_args( diff --git a/lib/spack/spack/test/cmd/audit.py b/lib/spack/spack/test/cmd/audit.py index 2bea59f9f46928..73bd534eead0bc 100644 --- a/lib/spack/spack/test/cmd/audit.py +++ b/lib/spack/spack/test/cmd/audit.py @@ -41,8 +41,7 @@ def test_audit_packages_https(mutable_config, mock_packages, monkeypatch): # Without providing --all should fail audit("packages-https", fail_on_error=False) - # The mock configuration has duplicate definitions of some compilers - assert audit.returncode == 1 + assert audit.returncode == 2 # This uses http and should fail audit("packages-https", "test-dependency", fail_on_error=False) diff --git a/lib/spack/spack/test/cmd/buildcache.py b/lib/spack/spack/test/cmd/buildcache.py index 4090d0e4037750..4e30908694a3b5 100644 --- a/lib/spack/spack/test/cmd/buildcache.py +++ b/lib/spack/spack/test/cmd/buildcache.py @@ -308,7 +308,7 @@ def manifest_insert(manifest, spec, dest_url): # Trigger the warning output = buildcache("sync", "--manifest-glob", manifest_file, "dest", "ignored") - assert "Ignoring unused arguemnt: ignored" in output + assert "Ignoring unused argument: ignored" in output verify_mirror_contents() shutil.rmtree(str(dest_mirror_dir)) @@ -360,6 +360,21 @@ def test_buildcache_create_install( cache_entry.destroy() +def _mock_uploader(tmp_path: pathlib.Path): + class DontUpload(spack.binary_distribution.Uploader): + def __init__(self): + super().__init__( + spack.mirrors.mirror.Mirror.from_local_path(str(tmp_path)), False, False + ) + self.pushed = [] + + def push(self, specs: List[spack.spec.Spec]): + self.pushed.extend(s.name for s in specs) + return [], [] + + return DontUpload() + + @pytest.mark.parametrize( "things_to_install,expected", [ @@ -411,18 +426,7 @@ def test_correct_specs_are_pushed( PackageInstaller([spec.package], explicit=True, fake=True).install() slash_hash = f"/{spec.dag_hash()}" - class DontUpload(spack.binary_distribution.Uploader): - def __init__(self): - super().__init__( - spack.mirrors.mirror.Mirror.from_local_path(str(tmp_path)), False, False - ) - self.pushed = [] - - def push(self, specs: List[spack.spec.Spec]): - self.pushed.extend(s.name for s in specs) - return [], [] # nothing skipped, nothing errored - - uploader = DontUpload() + uploader = _mock_uploader(tmp_path) monkeypatch.setattr( spack.binary_distribution, "make_uploader", lambda *args, **kwargs: uploader @@ -511,6 +515,26 @@ def test_best_effort_vs_fail_fast_when_dep_not_installed(tmp_path: pathlib.Path, assert set(specs) == {s for s in mpileaks.traverse() if s.name != "mpich"} +def test_allow_missing_when_dep_not_installed(tmp_path: pathlib.Path, mutable_database): + """When --allow-missing is passed, the push command should push installed specs and skip specs + that are not installed without raising an error.""" + + mirror("add", "--unsigned", "my-mirror", str(tmp_path)) + + # Uninstall mpich so that its dependent mpileaks can't be pushed + for s in mutable_database.query_local("mpich"): + s.package.do_uninstall(force=True) + + # There should be warnings but no errors + buildcache("push", "--update-index", "--allow-missing", "my-mirror", "mpileaks^mpich") + + specs = spack.binary_distribution.update_cache_and_get_specs() + + # Everything but mpich should be pushed + mpileaks = mutable_database.query_local("mpileaks^mpich")[0] + assert set(specs) == {s for s in mpileaks.traverse() if s.name != "mpich"} + + def test_push_without_build_deps( tmp_path: pathlib.Path, temporary_store, mock_packages, mutable_config ): @@ -556,7 +580,9 @@ def test_check_mirror_for_layout(v2_buildcache_layout, mutable_config, capfd): assert all([word in err for word in ["Warning", "missing", "layout"]]) -def test_url_buildcache_entry_v2_exists(v2_buildcache_layout, mock_packages, mutable_config): +def test_url_buildcache_entry_v2_exists( + v2_buildcache_layout, mock_packages, mutable_config, do_not_check_runtimes_on_reuse +): """Test existence check for v2 buildcache entries""" test_mirror_path = v2_buildcache_layout("unsigned") mirror_url = pathlib.Path(test_mirror_path).as_uri() @@ -603,6 +629,7 @@ def test_install_v2_layout( mutable_mock_env_path, install_mockery, mock_gnupghome, + do_not_check_runtimes_on_reuse, ): """Ensure we can still install from signed and unsigned v2 buildcache""" test_mirror_path = v2_buildcache_layout(signing) @@ -987,7 +1014,7 @@ def create_env_from_concrete_spec(spec: spack.spec.Spec): e = ev.environment_from_name_or_dir(env_name) with e: add(f"{spec.name}/{spec.dag_hash()}") - # This should handle updating the environment to mark all packges as installed + # This should handle updating the environment to mark all packages as installed install() return e @@ -1019,7 +1046,7 @@ def read_specs_in_index(mirror_directory, view): mirror_metadata = spack.binary_distribution.MirrorMetadata( f"file://{mirror_directory}", spack.mirrors.mirror.SUPPORTED_LAYOUT_VERSIONS[0], view ) - fetcher = spack.binary_distribution.DefaultIndexFetcher(mirror_metadata, None) + fetcher = spack.binary_distribution.DefaultIndexHandler(mirror_metadata, None) result = fetcher.conditional_fetch() db_dict = json.loads(result.data) return set([h for h in db_dict["database"]["installs"]]) @@ -1055,7 +1082,7 @@ def test_buildcache_create_view_empty( with pytest.raises(spack.binary_distribution.FetchIndexError): hashes_in_view = read_specs_in_index(mirror_directory, "test_view") - # Write a minimal lockfile (this is not validated/read by an enviornment) + # Write a minimal lockfile (this is not validated/read by an environment) empty_manifest = tmp_path / "emptylock" / "spack.yaml" empty_manifest.parent.mkdir(exist_ok=False) empty_manifest.write_text("spack: {}", encoding="utf-8") @@ -1318,3 +1345,151 @@ def test_buildcache_check_index_full( assert "The index blob is missing" in out assert "Unindexed specs: 15" in out assert "Missing blobs: 1" + + +def test_buildcache_push_with_group( + tmp_path: pathlib.Path, monkeypatch, install_mockery, mock_fetch, mutable_mock_env_path +): + """Tests that --group pushes only specs from the requested group.""" + env_dir = tmp_path / "myenv" + env_dir.mkdir() + (env_dir / "spack.yaml").write_text( + """\ +spack: + specs: + - libelf + - group: extra + specs: + - libdwarf + view: false +""" + ) + + mirror_dir = tmp_path / "mirror" + mirror_dir.mkdir() + + uploader = _mock_uploader(mirror_dir) + monkeypatch.setattr( + spack.binary_distribution, "make_uploader", lambda *args, **kwargs: uploader + ) + + with ev.Environment(env_dir) as e: + e.concretize() + e.write() + for _, root in e.concretized_specs(): + PackageInstaller([root.package], explicit=True, fake=True).install() + + buildcache("push", "--unsigned", "--only", "package", "--group", "extra", str(mirror_dir)) + + assert uploader.pushed == ["libdwarf"] + + +def test_buildcache_push_with_multiple_groups( + tmp_path: pathlib.Path, monkeypatch, install_mockery, mock_fetch, mutable_mock_env_path +): + """Tests that --group can be repeated to push specs from multiple groups.""" + env_dir = tmp_path / "myenv" + env_dir.mkdir() + (env_dir / "spack.yaml").write_text( + """\ +spack: + specs: + - libelf + - group: extra + specs: + - libdwarf + view: false +""" + ) + + mirror_dir = tmp_path / "mirror" + mirror_dir.mkdir() + + uploader = _mock_uploader(mirror_dir) + monkeypatch.setattr( + spack.binary_distribution, "make_uploader", lambda *args, **kwargs: uploader + ) + + with ev.Environment(env_dir) as e: + e.concretize() + e.write() + for _, root in e.concretized_specs(): + PackageInstaller([root.package], explicit=True, fake=True).install() + + buildcache( + "push", + "--unsigned", + "--only", + "package", + "--group", + "default", + "--group", + "extra", + str(mirror_dir), + ) + + assert set(uploader.pushed) == {"libelf", "libdwarf"} + assert len(uploader.pushed) == len(set(uploader.pushed)) + + +def test_buildcache_push_group_nonexistent_errors(tmp_path: pathlib.Path, mutable_mock_env_path): + """Tests that --group with a nonexistent group name raises an error.""" + env_dir = tmp_path / "myenv" + env_dir.mkdir() + (env_dir / "spack.yaml").write_text( + """\ +spack: + specs: + - libelf + view: false +""" + ) + + mirror_dir = tmp_path / "mirror" + mirror_dir.mkdir() + + with ev.Environment(env_dir): + with pytest.raises(spack.main.SpackCommandError): + buildcache( + "push", "--unsigned", "--group", "nonexistent", str(mirror_dir), fail_on_error=True + ) + + +def test_buildcache_push_group_and_specs_mutually_exclusive( + tmp_path: pathlib.Path, mutable_mock_env_path +): + """Tests that --group and explicit specs on the command line are mutually exclusive.""" + env_dir = tmp_path / "myenv" + env_dir.mkdir() + (env_dir / "spack.yaml").write_text( + """\ +spack: + specs: + - libelf + view: false +""" + ) + + mirror_dir = tmp_path / "mirror" + mirror_dir.mkdir() + + with ev.Environment(env_dir): + with pytest.raises(spack.main.SpackCommandError): + buildcache( + "push", + "--unsigned", + "--group", + "default", + str(mirror_dir), + "libelf", + fail_on_error=True, + ) + + +def test_buildcache_push_group_requires_active_env(tmp_path: pathlib.Path): + """Tests that ck--group without an active environment produces an error.""" + mirror_dir = tmp_path / "mirror" + mirror_dir.mkdir() + + with pytest.raises(spack.main.SpackCommandError): + buildcache("push", "--unsigned", "--group", "default", str(mirror_dir), fail_on_error=True) diff --git a/lib/spack/spack/test/cmd/checksum.py b/lib/spack/spack/test/cmd/checksum.py index fa676aab11023b..de9e4e87aa7d25 100644 --- a/lib/spack/spack/test/cmd/checksum.py +++ b/lib/spack/spack/test/cmd/checksum.py @@ -366,6 +366,7 @@ def install(self, spec, prefix): version("1.2.5", sha256="abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890") version("1.2.3", sha256="1795c7d067a43174113fdf03447532f373e1c6c57c08d61d9e4e9be5e244b05e") """ + # ruff: disable[E501] # two new versions are added assert spack.cmd.checksum.add_versions_to_pkg(str(pkg_path), version_lines) == 2 assert ( @@ -388,3 +389,6 @@ def install(self, spec, prefix): make("install") """ ) + + +# ruff: enable[E501] diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index 5255aa838d72de..c111bbba4728e5 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -200,7 +200,8 @@ def test_ci_generate_with_env(ci_generate_test, tmp_path: pathlib.Path, mock_bin assert "rebuild-index" in yaml_contents rebuild_job = yaml_contents["rebuild-index"] assert ( - rebuild_job["script"][0] == f"spack buildcache update-index --keys {mirror_url.as_uri()}" + rebuild_job["script"][0] + == f"spack -v buildcache update-index --keys {mirror_url.as_uri()}" ) assert rebuild_job["custom_attribute"] == "custom!" @@ -338,6 +339,7 @@ def test_ci_generate_with_custom_settings( "spack -d ci rebuild", "cd ENV", "spack env activate --without-view .", + "spack spec /$SPACK_JOB_SPEC_DAG_HASH", "spack ci rebuild", ] assert ci_obj["after_script"] == ["rm -rf /some/path/spack"] @@ -876,7 +878,7 @@ def test_push_to_build_cache( # Validate resulting buildcache (database) index layout_version = spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION mirror_metadata = spack.binary_distribution.MirrorMetadata(mirror_url, layout_version) - index_fetcher = spack.binary_distribution.DefaultIndexFetcher(mirror_metadata, None) + index_fetcher = spack.binary_distribution.DefaultIndexHandler(mirror_metadata, None) result = index_fetcher.conditional_fetch() spack.vendor.jsonschema.validate(json.loads(result.data), db_idx_schema) @@ -1026,7 +1028,7 @@ def test_ci_generate_override_runner_attrs( assert the_elt["after_script"][0] == "post step one" if "dependent-install" in ci_key: # The dependent-install match specifies that we keep the two - # top level variables, but add a third specifc one. It + # top level variables, but add a third specific one. It # also adds a custom tag which should be combined with # the top-level tag. the_elt = yaml_contents[ci_key] @@ -1047,7 +1049,12 @@ def test_ci_generate_override_runner_attrs( def test_ci_rebuild_index( - tmp_path: pathlib.Path, working_env, mutable_mock_env_path, install_mockery, mock_fetch + tmp_path: pathlib.Path, + working_env, + mutable_mock_env_path, + install_mockery, + mock_fetch, + mock_binary_index, ): scratch = tmp_path / "working_dir" mirror_dir = scratch / "mirror" diff --git a/lib/spack/spack/test/cmd/commands.py b/lib/spack/spack/test/cmd/commands.py index ffd5547795ed82..c8362b539bc2ea 100644 --- a/lib/spack/spack/test/cmd/commands.py +++ b/lib/spack/spack/test/cmd/commands.py @@ -65,7 +65,7 @@ def test_subcommands(): def test_alias_overrides_builtin(mutable_config: spack.config.Configuration, capfd): - """Test that spack commands cannot be overriden by aliases.""" + """Test that spack commands cannot be overridden by aliases.""" mutable_config.set("config:aliases", {"install": "find"}) cmd, args = spack.main.resolve_alias("install", ["install", "-v"]) assert cmd == "install" and args == ["install", "-v"] @@ -199,6 +199,14 @@ def test_bash_completion(): assert "_spack_compiler_add() {" in out2 +def test_bash_completion_choices(): + """Test that bash completion includes choices for positional arguments.""" + out = commands("--format=bash") + + # `spack env view` has a positional `action` with choices + assert 'SPACK_COMPREPLY="disable enable regenerate"' in out + + def test_fish_completion(): """Test the fish completion writer.""" out1 = commands("--format=fish") @@ -297,4 +305,15 @@ def test_updated_completion_scripts(shell, tmp_path: pathlib.Path): commands("--aliases", "--format", shell, "--header", header, "--update", new_script) - assert filecmp.cmp(old_script, new_script), msg + if not filecmp.cmp(old_script, new_script): + # If there is a diff, something is wrong: in that case output what the diff is. + import difflib + + with open(old_script, "r", encoding="utf-8") as f1, open( + new_script, "r", encoding="utf-8" + ) as f2: + l1 = f1.readlines() + l2 = f2.readlines() + diff = difflib.unified_diff(l1, l2, fromfile=old_script, tofile=new_script) + msg += "\nDiff failure:\n\n" + "".join(diff) + raise AssertionError(msg) diff --git a/lib/spack/spack/test/cmd/common/spec_strings.py b/lib/spack/spack/test/cmd/common/spec_strings.py new file mode 100644 index 00000000000000..f1ea2ea89d6a17 --- /dev/null +++ b/lib/spack/spack/test/cmd/common/spec_strings.py @@ -0,0 +1,112 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import pathlib + +import spack.cmd.common.spec_strings + + +def test_spec_strings(tmp_path: pathlib.Path): + (tmp_path / "example.py").write_text( + """\ +def func(x): + print("dont fix %s me" % x, 3) + return x.satisfies("+foo %gcc +bar") and x.satisfies("%gcc +baz") +""" + ) + (tmp_path / "example.json").write_text( + """\ +{ + "spec": [ + "+foo %gcc +bar~nope ^dep %clang +yup @3.2 target=x86_64 /abcdef ^another %gcc ", + "%gcc +baz" + ], + "%gcc x=y": 2 +} +""" + ) + (tmp_path / "example.yaml").write_text( + """\ +spec: + - "+foo %gcc +bar" + - "%gcc +baz" + - "this is fine %clang" +"%gcc x=y": 2 +""" + ) + + issues = set() + + def collect_issues(path: str, line: int, col: int, old: str, new: str): + issues.add((path, line, col, old, new)) + + # check for issues with custom handler + spack.cmd.common.spec_strings._check_spec_strings( + [ + str(tmp_path / "nonexistent.py"), + str(tmp_path / "example.py"), + str(tmp_path / "example.json"), + str(tmp_path / "example.yaml"), + ], + handler=collect_issues, + ) + + assert issues == { + ( + str(tmp_path / "example.json"), + 3, + 9, + "+foo %gcc +bar~nope ^dep %clang +yup @3.2 target=x86_64 /abcdef ^another %gcc ", + "+foo +bar~nope %gcc ^dep +yup @3.2 target=x86_64 /abcdef %clang ^another %gcc ", + ), + (str(tmp_path / "example.json"), 4, 9, "%gcc +baz", "+baz %gcc"), + (str(tmp_path / "example.json"), 6, 5, "%gcc x=y", "x=y %gcc"), + (str(tmp_path / "example.py"), 3, 23, "+foo %gcc +bar", "+foo +bar %gcc"), + (str(tmp_path / "example.py"), 3, 57, "%gcc +baz", "+baz %gcc"), + (str(tmp_path / "example.yaml"), 2, 5, "+foo %gcc +bar", "+foo +bar %gcc"), + (str(tmp_path / "example.yaml"), 3, 5, "%gcc +baz", "+baz %gcc"), + (str(tmp_path / "example.yaml"), 5, 1, "%gcc x=y", "x=y %gcc"), + } + + # fix the issues in the files + spack.cmd.common.spec_strings._check_spec_strings( + [ + str(tmp_path / "nonexistent.py"), + str(tmp_path / "example.py"), + str(tmp_path / "example.json"), + str(tmp_path / "example.yaml"), + ], + handler=spack.cmd.common.spec_strings._spec_str_fix_handler, + ) + + assert ( + (tmp_path / "example.json").read_text() + == """\ +{ + "spec": [ + "+foo +bar~nope %gcc ^dep +yup @3.2 target=x86_64 /abcdef %clang ^another %gcc ", + "+baz %gcc" + ], + "x=y %gcc": 2 +} +""" + ) + assert ( + (tmp_path / "example.py").read_text() + == """\ +def func(x): + print("dont fix %s me" % x, 3) + return x.satisfies("+foo +bar %gcc") and x.satisfies("+baz %gcc") +""" + ) + assert ( + (tmp_path / "example.yaml").read_text() + == """\ +spec: + - "+foo +bar %gcc" + - "+baz %gcc" + - "this is fine %clang" +"x=y %gcc": 2 +""" + ) diff --git a/lib/spack/spack/test/cmd/compiler.py b/lib/spack/spack/test/cmd/compiler.py index deeec78e16595c..361a1a1fcbc112 100644 --- a/lib/spack/spack/test/cmd/compiler.py +++ b/lib/spack/spack/test/cmd/compiler.py @@ -169,7 +169,9 @@ def test_compiler_find_prefer_no_suffix(no_packages_yaml, working_env, compilers @pytest.mark.not_on_windows("Cannot execute bash script on Windows") def test_compiler_find_path_order(no_packages_yaml, working_env, compilers_dir): - """Ensure that we look for compilers in the same order as PATH, when there are duplicates""" + """When the same compiler version is found in two PATH directories, only the first + entry in PATH is kept and a warning is emitted for the duplicate. + """ new_dir = compilers_dir / "first_in_path" new_dir.mkdir() for name in ("gcc-8", "g++-8", "gfortran-8"): @@ -177,13 +179,14 @@ def test_compiler_find_path_order(no_packages_yaml, working_env, compilers_dir): # Set PATH to have the new folder searched first os.environ["PATH"] = f"{str(new_dir)}:{str(compilers_dir)}" - compiler("find", "--scope=site") + with pytest.warns(UserWarning, match="gcc@"): + compiler("find", "--scope=site") compilers = spack.compilers.config.all_compilers(scope="site") gcc = [x for x in compilers if x.satisfies("gcc@8.4")] - # Ensure we found both duplicates - assert len(gcc) == 2 + # Duplicate is dropped. Only the first entry in PATH is kept + assert len(gcc) == 1 assert gcc[0].extra_attributes["compilers"] == { "c": str(new_dir / "gcc-8"), "cxx": str(new_dir / "g++-8"), diff --git a/lib/spack/spack/test/cmd/concretize.py b/lib/spack/spack/test/cmd/concretize.py index f4b61aa67801c5..cc154c7435312c 100644 --- a/lib/spack/spack/test/cmd/concretize.py +++ b/lib/spack/spack/test/cmd/concretize.py @@ -20,36 +20,40 @@ @pytest.mark.parametrize("unify", unification_strategies) -def test_concretize_all_test_dependencies(unify, mutable_mock_env_path): +def test_concretize_all_test_dependencies(unify, mutable_config, mutable_mock_env_path): """Check all test dependencies are concretized.""" env("create", "test") with ev.read("test") as e: - e.unify = unify + mutable_config.set("concretizer:unify", unify) add("depb") concretize("--test", "all") assert e.matching_spec("test-dependency") @pytest.mark.parametrize("unify", unification_strategies) -def test_concretize_root_test_dependencies_not_recursive(unify, mutable_mock_env_path): +def test_concretize_root_test_dependencies_not_recursive( + unify, mutable_config, mutable_mock_env_path +): """Check that test dependencies are not concretized recursively.""" env("create", "test") with ev.read("test") as e: - e.unify = unify + mutable_config.set("concretizer:unify", unify) add("depb") concretize("--test", "root") assert e.matching_spec("test-dependency") is None @pytest.mark.parametrize("unify", unification_strategies) -def test_concretize_root_test_dependencies_are_concretized(unify, mutable_mock_env_path): +def test_concretize_root_test_dependencies_are_concretized( + unify, mutable_config, mutable_mock_env_path +): """Check that root test dependencies are concretized.""" env("create", "test") with ev.read("test") as e: - e.unify = unify + mutable_config.set("concretizer:unify", unify) add("pkg-a") add("pkg-b") concretize("--test", "root") diff --git a/lib/spack/spack/test/cmd/config.py b/lib/spack/spack/test/cmd/config.py index 406d1bbeaf1dce..c84fc45ce8275c 100644 --- a/lib/spack/spack/test/cmd/config.py +++ b/lib/spack/spack/test/cmd/config.py @@ -6,9 +6,11 @@ import os import pathlib import re +import shutil import pytest +import spack.cmd.config as config_cmd import spack.concretize import spack.config import spack.database @@ -67,7 +69,7 @@ def test_config_scopes(path, types, mutable_mock_env_path): assert "command_line" in output assert "_builtin" in output if types: - if not any(i in ("all", "path") for i in types): + if not any(i in ("all", "path", "include") for i in types): assert "site" not in output if not any(i in ("all", "env", "include", "path") for i in types): assert not output or all(":" not in x for x in output) @@ -111,7 +113,7 @@ def test_include_overrides(mutable_config): mutable_config.push_scope(spack.config.InternalConfigScope("override", {"include:": []})) - # overridden scopes are not shown wtihout `-v` + # overridden scopes are not shown without `-v` output = config("scopes").strip() lines = output.split("\n") assert "user" not in lines @@ -121,7 +123,7 @@ def test_include_overrides(mutable_config): # scopes with ConfigScopePriority.DEFAULTS remain assert "_builtin" in lines - # overridden scopes are shown wtih `-v` and marked 'override' + # overridden scopes are shown with `-v` and marked 'override' output = config("scopes", "-v").strip() lines = output.split("\n") assert "override" in next(line for line in lines if line.startswith("user")) @@ -133,21 +135,21 @@ def test_blame_override(mutable_config): # includes are present when section is specified output = config("blame", "include").strip() include_path = re.escape(os.path.join(mutable_config.scopes["site"].path, "include.yaml")) - assert re.search(rf"include:\n{include_path}:\d+\s+\- path: base", output) + assert re.search(rf"{include_path}:\d+\s+\- path: base", output) # includes are also present when section is NOT specified output = config("blame").strip() - assert re.search(rf"include:\n{include_path}:\d+\s+\- path: base", output) + assert re.search(rf"{include_path}:\d+\s+\- path: base", output) mutable_config.push_scope(spack.config.InternalConfigScope("override", {"include:": []})) # site includes are not present when overridden output = config("blame", "include").strip() - assert not re.search(rf"include:\n{include_path}:\d+\s+\- path: base", output) + assert not re.search(rf"{include_path}:\d+\s+\- path: base", output) assert "include: []" in output output = config("blame").strip() - assert not re.search(rf"include:\n{include_path}:\d+\s+\- path: base", output) + assert not re.search(rf"{include_path}:\d+\s+\- path: base", output) assert "include: []" in output @@ -718,3 +720,74 @@ def update_config(data): with ev.Environment(str(tmp_path)) as e: assert not e.manifest.yaml_content["spack"]["config"]["ccache"] + + +_GROUP_OVERRIDE_SPACK_YAML = """\ +spack: + specs: + - group: mygroup + specs: + - zlib + override: + packages: + zlib: + version: ['1.2.13'] +""" + + +@pytest.mark.parametrize("cmd_str", ["get", "blame"]) +def test_config_with_group_shows_override_packages(cmd_str, tmp_path, mutable_config): + """Tests that packages should show that group's override packages config, + when the option is given. + """ + (tmp_path / "spack.yaml").write_text(_GROUP_OVERRIDE_SPACK_YAML) + + with ev.Environment(str(tmp_path)): + output = config(cmd_str, "packages") + assert "1.2.13" not in output + if cmd_str == "blame": + assert "env:groups:mygroup" not in output + output = config(cmd_str, "--group=mygroup", "packages") + assert "1.2.13" in output + if cmd_str == "blame": + assert "env:groups:mygroup" in output + + +@pytest.mark.parametrize("cmd_str", ["get", "blame"]) +def test_config_with_group_requires_active_environment(cmd_str, mutable_config): + """Tests that using groups outside an environment should give a clear error.""" + output = config(cmd_str, "--group=mygroup", "packages", fail_on_error=False) + assert config.returncode == 2 + assert "--group requires an active environment" in output + + +@pytest.mark.parametrize("cmd_str", ["get", "blame"]) +def test_config_with_unknown_group_gives_clear_error(cmd_str, tmp_path, mutable_config): + """Tests that using a non-existing group gives a clear error.""" + (tmp_path / "spack.yaml").write_text("spack:\n specs:\n - zlib\n") + with ev.Environment(str(tmp_path)): + output = config(cmd_str, "--group=nonexistent", "packages", fail_on_error=False) + assert config.returncode == 1 + assert "'nonexistent' not found in" in output + + +@pytest.mark.regression("52152") +def test_config_edit_creates_scope_dir(mutable_config, working_env, monkeypatch): + """Tests that `spack config edit` can create the scope directory if it does not exist.""" + scope_name = spack.config.default_modify_scope("config") + scope_dir = pathlib.Path(mutable_config.scopes[scope_name].path) + + # Remove the scope directory to simulate a "fresh start" with no ~/.spack + shutil.rmtree(scope_dir) + assert not scope_dir.exists() + + editor_called = [] + + def fake_editor(*args, **kwargs): + editor_called.extend(args) + + monkeypatch.setattr(config_cmd, "editor", fake_editor) + config("edit", "config") + + assert scope_dir.exists(), "scope directory should be created before invoking the editor" + assert editor_called, "editor should have been called" diff --git a/lib/spack/spack/test/cmd/create.py b/lib/spack/spack/test/cmd/create.py index d521d47e6ce9cd..ae619c03eb8ea6 100644 --- a/lib/spack/spack/test/cmd/create.py +++ b/lib/spack/spack/test/cmd/create.py @@ -150,7 +150,7 @@ def test_create_template_bad_name(mock_test_repo, name, expected): """Test template creation with bad name options.""" output = create("--skip-editor", "-n", name, fail_on_error=False) assert expected in output - assert create.returncode != 0 + assert create.returncode == 1 def test_build_system_guesser_no_stage(): diff --git a/lib/spack/spack/test/cmd/deconcretize.py b/lib/spack/spack/test/cmd/deconcretize.py index 4c72a8ca4ea0f7..fdceed7d73f5ca 100644 --- a/lib/spack/spack/test/cmd/deconcretize.py +++ b/lib/spack/spack/test/cmd/deconcretize.py @@ -43,7 +43,7 @@ def test_deconcretize_root(test_env): with ev.read("test") as e: output = deconcretize("-y", "--root", "pkg-b@1.0") assert "No matching specs to deconcretize" in output - assert len(e.concretized_order) == 2 + assert len(e.concretized_roots) == 2 deconcretize("-y", "--root", "pkg-a@2.0") specs = [s for s, _ in e.concretized_specs()] @@ -59,7 +59,7 @@ def test_deconcretize_all_root(test_env): output = deconcretize("-y", "--root", "--all", "pkg-b") assert "No matching specs to deconcretize" in output - assert len(e.concretized_order) == 2 + assert len(e.concretized_roots) == 2 deconcretize("-y", "--root", "--all", "pkg-a") specs = [s for s, _ in e.concretized_specs()] diff --git a/lib/spack/spack/test/cmd/dependencies.py b/lib/spack/spack/test/cmd/dependencies.py index 14925e6523f805..5447436aff0983 100644 --- a/lib/spack/spack/test/cmd/dependencies.py +++ b/lib/spack/spack/test/cmd/dependencies.py @@ -20,8 +20,9 @@ "multi-provider-mpi", "zmpi", ] -COMPILERS = ["gcc", "llvm"] +COMPILERS = ["gcc", "llvm", "compiler-with-deps"] MPI_DEPS = ["fake"] +COMPILER_DEPS = ["binutils-for-test", "zlib"] @pytest.mark.parametrize( @@ -30,7 +31,13 @@ (["mpileaks"], set(["callpath"] + MPIS + COMPILERS)), ( ["--transitive", "mpileaks"], - set(["callpath", "dyninst", "libdwarf", "libelf"] + MPIS + MPI_DEPS + COMPILERS), + set( + ["callpath", "dyninst", "libdwarf", "libelf"] + + MPIS + + MPI_DEPS + + COMPILERS + + COMPILER_DEPS + ), ), (["--transitive", "--deptype=link,run", "dtbuild1"], {"dtlink2", "dtrun2"}), (["--transitive", "--deptype=build", "dtbuild1"], {"dtbuild2", "dtlink2"}), diff --git a/lib/spack/spack/test/cmd/dev_build.py b/lib/spack/spack/test/cmd/dev_build.py index dba112b2691f06..4a01074a1fddea 100644 --- a/lib/spack/spack/test/cmd/dev_build.py +++ b/lib/spack/spack/test/cmd/dev_build.py @@ -44,7 +44,7 @@ def test_dev_build_basics(tmp_path: pathlib.Path, install_mockery): assert os.path.exists(str(tmp_path)) -def test_dev_build_before(tmp_path: pathlib.Path, install_mockery): +def test_dev_build_before(tmp_path: pathlib.Path, install_mockery, installer_variant): spec = spack.concretize.concretize_one( spack.spec.Spec(f"dev-build-test-install@0.0.0 dev_path={tmp_path}") ) @@ -62,7 +62,8 @@ def test_dev_build_before(tmp_path: pathlib.Path, install_mockery): assert not os.path.exists(spec.prefix) -def test_dev_build_until(tmp_path: pathlib.Path, install_mockery): +@pytest.mark.parametrize("last_phase", ["edit", "install"]) +def test_dev_build_until(tmp_path: pathlib.Path, install_mockery, last_phase, installer_variant): spec = spack.concretize.concretize_one( spack.spec.Spec(f"dev-build-test-install@0.0.0 dev_path={tmp_path}") ) @@ -71,7 +72,7 @@ def test_dev_build_until(tmp_path: pathlib.Path, install_mockery): with open(spec.package.filename, "w", encoding="utf-8") as f: # type: ignore f.write(spec.package.original_string) # type: ignore - dev_build("-u", "edit", "dev-build-test-install@0.0.0") + dev_build("--until", last_phase, "dev-build-test-install@0.0.0") assert spec.package.filename in os.listdir(os.getcwd()) # type: ignore with open(spec.package.filename, "r", encoding="utf-8") as f: # type: ignore @@ -81,28 +82,7 @@ def test_dev_build_until(tmp_path: pathlib.Path, install_mockery): assert not spack.store.STORE.db.query(spec, installed=True) -def test_dev_build_until_last_phase(tmp_path: pathlib.Path, install_mockery): - # Test that we ignore the last_phase argument if it is already last - spec = spack.concretize.concretize_one( - spack.spec.Spec(f"dev-build-test-install@0.0.0 dev_path={tmp_path}") - ) - - with fs.working_dir(str(tmp_path)): - with open(spec.package.filename, "w", encoding="utf-8") as f: - f.write(spec.package.original_string) - - dev_build("-u", "install", "dev-build-test-install@0.0.0") - - assert spec.package.filename in os.listdir(os.getcwd()) - with open(spec.package.filename, "r", encoding="utf-8") as f: - assert f.read() == spec.package.replacement_string - - assert os.path.exists(spec.prefix) - assert spack.store.STORE.db.query(spec, installed=True) - assert os.path.exists(str(tmp_path)) - - -def test_dev_build_before_until(tmp_path: pathlib.Path, install_mockery): +def test_dev_build_before_until(tmp_path: pathlib.Path, install_mockery, installer_variant): spec = spack.concretize.concretize_one( spack.spec.Spec(f"dev-build-test-install@0.0.0 dev_path={tmp_path}") ) @@ -120,12 +100,14 @@ def test_dev_build_before_until(tmp_path: pathlib.Path, install_mockery): out = dev_build("-u", bad_phase, "dev-build-test-install@0.0.0", fail_on_error=False) assert bad_phase in out assert not_allowed in out - assert not_installed in out + if installer_variant == "old": + assert not_installed in out out = dev_build("-b", bad_phase, "dev-build-test-install@0.0.0", fail_on_error=False) assert bad_phase in out assert not_allowed in out - assert not_installed in out + if installer_variant == "old": + assert not_installed in out def _print_spack_short_spec(*args): @@ -179,7 +161,8 @@ def test_dev_build_fails_nonexistent_package_name(mock_packages): def test_dev_build_fails_no_version(mock_packages): output = dev_build("dev-build-test-install", fail_on_error=False) - assert "dev-build spec must have a single, concrete version" in output + assert "spec must have a single, concrete version" in output + assert dev_build.returncode == 2 def test_dev_build_can_parse_path_with_at_symbol(tmp_path: pathlib.Path, install_mockery): @@ -196,7 +179,9 @@ def test_dev_build_can_parse_path_with_at_symbol(tmp_path: pathlib.Path, install assert spec.package.filename in os.listdir(spec.prefix) -def test_dev_build_env(tmp_path: pathlib.Path, install_mockery, mutable_mock_env_path): +def test_dev_build_env( + tmp_path: pathlib.Path, install_mockery, mutable_mock_env_path, installer_variant +): """Test Spack does dev builds for packages in develop section of env.""" # setup dev-build-test-install package for dev build build_dir = tmp_path / "build" @@ -236,7 +221,7 @@ def test_dev_build_env(tmp_path: pathlib.Path, install_mockery, mutable_mock_env def test_dev_build_env_with_vars( - tmp_path: pathlib.Path, install_mockery, mutable_mock_env_path, monkeypatch + tmp_path: pathlib.Path, install_mockery, mutable_mock_env_path, monkeypatch, installer_variant ): """Test Spack does dev builds for packages in develop section of env (path with variables).""" # setup dev-build-test-install package for dev build @@ -279,7 +264,7 @@ def test_dev_build_env_with_vars( def test_dev_build_env_version_mismatch( - tmp_path: pathlib.Path, install_mockery, mutable_mock_env_path + tmp_path: pathlib.Path, install_mockery, mutable_mock_env_path, installer_variant ): """Test Spack constraints concretization by develop specs.""" # setup dev-build-test-install package for dev build diff --git a/lib/spack/spack/test/cmd/develop.py b/lib/spack/spack/test/cmd/develop.py index b029ca38b1abd3..b0146b6e99d0a4 100644 --- a/lib/spack/spack/test/cmd/develop.py +++ b/lib/spack/spack/test/cmd/develop.py @@ -330,14 +330,14 @@ def test_recursive(mutable_mock_env_path, install_mockery, mock_fetch): def test_develop_fails_with_multiple_concrete_versions( - mutable_mock_env_path, install_mockery, mock_fetch + mutable_mock_env_path, install_mockery, mock_fetch, mutable_config ): env("create", "test") with ev.read("test") as e: add("indirect-mpich@1.0") add("indirect-mpich@0.9") - e.unify = False + mutable_config.set("concretizer:unify", False) e.concretize() with pytest.raises(SpackError) as develop_error: diff --git a/lib/spack/spack/test/cmd/diff.py b/lib/spack/spack/test/cmd/diff.py index 1e703c4849f109..db28be8a2b5fa2 100644 --- a/lib/spack/spack/test/cmd/diff.py +++ b/lib/spack/spack/test/cmd/diff.py @@ -85,7 +85,7 @@ def test_diff_cmd(install_mockery, mock_fetch, mock_archive, mock_packages): # Calculate the comparison (c) c = spack.cmd.diff.compare_specs(specA, specB, to_string=True) - # these particular diffs should have the same length b/c thre aren't + # these particular diffs should have the same length b/c there aren't # any node differences -- just value differences. assert len(c["a_not_b"]) == len(c["b_not_a"]) diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index 8ea37d3997d7b4..327f2778703094 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -8,6 +8,7 @@ import os import pathlib import shutil +import sys from argparse import Namespace from typing import Any, Dict, Optional @@ -18,8 +19,6 @@ import spack.config import spack.environment as ev import spack.environment.depfile as depfile -import spack.environment.environment -import spack.environment.shell import spack.error import spack.llnl.util.filesystem as fs import spack.llnl.util.link_tree @@ -30,6 +29,7 @@ import spack.package_base import spack.paths import spack.repo +import spack.schema.env import spack.solver.asp import spack.stage import spack.store @@ -39,10 +39,12 @@ from spack.cmd.env import _env_create from spack.installer import PackageInstaller from spack.llnl.util.filesystem import readlink +from spack.llnl.util.lang import dedupe from spack.main import SpackCommand, SpackCommandError from spack.spec import Spec from spack.stage import stage_prefix from spack.test.conftest import RepoBuilder +from spack.traverse import traverse_nodes from spack.util.executable import Executable from spack.util.path import substitute_path_variables from spack.version import Version @@ -76,19 +78,18 @@ def setup_combined_multiple_env(): test1 = ev.read("test1") with test1: add("mpich@1.0") - test1.concretize() - test1.write() + test1.concretize() + test1.write() env("create", "test2") test2 = ev.read("test2") with test2: add("libelf") - test2.concretize() - test2.write() + test2.concretize() + test2.write() env("create", "--include-concrete", "test1", "--include-concrete", "test2", "combined_env") combined = ev.read("combined_env") - return test1, test2, combined @@ -253,6 +254,12 @@ def template_combinatorial_env(tmp_path: pathlib.Path): """ +def test_add_requires_active_env(): + """Test that spack add exits with code 2 when no environment is active.""" + add("hdf5", fail_on_error=False) + assert add.returncode == 2 + + def test_add(): e = ev.create("test") e.add("mpileaks") @@ -264,7 +271,6 @@ def test_change_match_spec(): e = ev.read("test") with e: - add("mpileaks@2.1") add("mpileaks@2.2") @@ -291,14 +297,12 @@ def test_change_multiple_matches(): def test_env_add_virtual(): env("create", "test") - e = ev.read("test") e.add("mpi") e.concretize() - hashes = e.concretized_order - assert len(hashes) == 1 - spec = e.specs_by_hash[hashes[0]] + assert len(e.concretized_roots) == 1 + spec = e.specs_by_hash[e.concretized_roots[0].hash] assert spec.intersects("mpi") @@ -475,8 +479,9 @@ def test_concretize(): e = ev.create("test") e.add("mpileaks") e.concretize() - env_specs = e._get_environment_specs() - assert any(x.name == "mpileaks" for x in env_specs) + + assert len(e.concretized_roots) == 1 + assert e.concretized_roots[0].root == Spec("mpileaks") def test_env_specs_partition(install_mockery, mock_fetch): @@ -512,12 +517,11 @@ def test_env_install_all(install_mockery, mock_fetch): e.add("cmake-client") e.concretize() e.install_all(fake=True) - env_specs = e._get_environment_specs() - spec = next(x for x in env_specs if x.name == "cmake-client") + spec = next(x for x in e.all_specs_generator() if x.name == "cmake-client") assert spec.installed -def test_env_install_single_spec(install_mockery, mock_fetch): +def test_env_install_single_spec(install_mockery, mock_fetch, installer_variant): env("create", "test") install = SpackCommand("install") @@ -526,46 +530,56 @@ def test_env_install_single_spec(install_mockery, mock_fetch): install("--fake", "--add", "cmake-client") e = ev.read("test") - assert e.user_specs[0].name == "cmake-client" - assert e.concretized_user_specs[0].name == "cmake-client" - assert e.specs_by_hash[e.concretized_order[0]].name == "cmake-client" + assert len(e.concretized_roots) == 1 + + item = e.concretized_roots[0] + assert list(e.user_specs) == [Spec("cmake-client")] + assert item.root == Spec("cmake-client") + assert e.specs_by_hash[item.hash].name == "cmake-client" @pytest.mark.parametrize("unify", [True, False, "when_possible"]) -def test_env_install_include_concrete_env(unify, install_mockery, mock_fetch, mutable_config): +@pytest.mark.parametrize("reuse", [True, False]) +def test_env_install_include_concrete_env( + unify, reuse, install_mockery, mock_fetch, mutable_config +): test1, test2, combined = setup_combined_multiple_env() - combined.unify = unify - if not unify: + if unify is False: combined.manifest.set_default_view(False) - combined.add("mpileaks") - combined.concretize() - combined.write() - with combined: + mutable_config.set("concretizer:unify", unify) + mutable_config.set("concretizer:reuse", reuse) + combined.add("mpileaks") + combined.concretize() + combined.write() install("--fake") - test1_roots = test1.concretized_order - test2_roots = test2.concretized_order - combined_included_roots = combined.included_concretized_order + test1_user_spec_hashes = [x.hash for x in test1.concretized_roots] + test2_user_spec_hashes = [x.hash for x in test2.concretized_roots] for spec in combined.all_specs(): assert spec.installed - assert test1_roots == combined_included_roots[test1.path] - assert test2_roots == combined_included_roots[test2.path] + assert test1_user_spec_hashes == [ + x.hash for x in combined.included_concretized_roots[test1.path] + ] + assert test2_user_spec_hashes == [ + x.hash for x in combined.included_concretized_roots[test2.path] + ] - mpileaks = combined.specs_by_hash[combined.concretized_order[0]] - if unify: - assert mpileaks["mpi"].dag_hash() in test1_roots - assert mpileaks["libelf"].dag_hash() in test2_roots - else: + mpileaks_hash = combined.concretized_roots[0].hash + mpileaks = combined.specs_by_hash[mpileaks_hash] + if unify is False and reuse is False: # check that unification is not by accident - assert mpileaks["mpi"].dag_hash() not in test1_roots + assert mpileaks["mpi"].dag_hash() not in test1_user_spec_hashes + else: + assert mpileaks["mpi"].dag_hash() in test1_user_spec_hashes + assert mpileaks["libelf"].dag_hash() in test2_user_spec_hashes -def test_env_roots_marked_explicit(install_mockery, mock_fetch): +def test_env_roots_marked_explicit(install_mockery, mock_fetch, installer_variant): install = SpackCommand("install") install("--fake", "dependent-install") @@ -616,7 +630,7 @@ def test_activate_adds_transitive_run_deps_to_path(install_mockery, mock_fetch, install("--add", "--fake", "depends-on-run-env") env_variables = {} - spack.environment.shell.activate(e).apply_modifications(env_variables) + ev.shell.activate(e).apply_modifications(env_variables) assert env_variables["DEPENDENCY_ENV_VAR"] == "1" @@ -687,29 +701,28 @@ def test_remove_after_concretize(): e.remove("mpileaks") assert Spec("mpileaks") not in e.user_specs - env_specs = e._get_environment_specs() - assert any(s.name == "mpileaks" for s in env_specs) + assert any(s.name == "mpileaks" for s in e.all_specs_generator()) e.add("mpileaks") assert any(s.name == "mpileaks" for s in e.user_specs) e.remove("mpileaks", force=True) assert Spec("mpileaks") not in e.user_specs - env_specs = e._get_environment_specs() - assert not any(s.name == "mpileaks" for s in env_specs) - - -def test_remove_before_concretize(): - e = ev.create("test") - e.unify = True + assert not any(s.name == "mpileaks" for s in e.all_specs_generator()) - e.add("mpileaks") - e.concretize() - e.remove("mpileaks") - e.concretize() +def test_remove_before_concretize(mutable_config): + """Tests the effect of concretization after adding and removing specs""" + with ev.create("test") as e: + mutable_config.set("concretizer:unify", True) + e.add("mpileaks") + e.concretize() + assert len(e.concretized_roots) == 1 + assert e.concrete_roots()[0].satisfies("mpileaks") - assert not list(e.concretized_specs()) + e.remove("mpileaks") + e.concretize() + assert not e.concretized_roots def test_remove_command(): @@ -938,9 +951,7 @@ def test_user_removed_spec(environment_from_manifest): after.write() read = ev.read("test") - env_specs = read._get_environment_specs() - - assert not any(x.name == "hypre" for x in env_specs) + assert not any(x.name == "hypre" for x in read.all_specs_generator()) def test_lockfile_spliced_specs(environment_from_manifest, install_mockery): @@ -995,12 +1006,10 @@ def test_init_from_lockfile(environment_from_manifest): for s1, s2 in zip(e1.user_specs, e2.user_specs): assert s1 == s2 - for h1, h2 in zip(e1.concretized_order, e2.concretized_order): - assert h1 == h2 - assert e1.specs_by_hash[h1] == e2.specs_by_hash[h2] + for r1, r2 in zip(e1.concretized_roots, e2.concretized_roots): + assert r1 == r2 - for s1, s2 in zip(e1.concretized_user_specs, e2.concretized_user_specs): - assert s1 == s2 + assert e1.specs_by_hash == e2.specs_by_hash def test_init_from_yaml(environment_from_manifest): @@ -1022,8 +1031,7 @@ def test_init_from_yaml(environment_from_manifest): for s1, s2 in zip(e1.user_specs, e2.user_specs): assert s1 == s2 - assert not e2.concretized_order - assert not e2.concretized_user_specs + assert not e2.concretized_roots assert not e2.specs_by_hash @@ -1058,8 +1066,7 @@ def test_init_from_env(use_name, environment_from_manifest): for s1, s2 in zip(e1.user_specs, e2.user_specs): assert s1 == s2 - assert e2.concretized_order == e1.concretized_order - assert e2.concretized_user_specs == e1.concretized_user_specs + assert e2.concretized_roots == e1.concretized_roots assert e2.specs_by_hash == e1.specs_by_hash assert os.path.exists(os.path.join(e2.path, "libelf")) @@ -1159,9 +1166,7 @@ def test_env_view_external_prefix(tmp_path: pathlib.Path, mutable_database, mock - spec: pkg-a@2.0 prefix: {a_prefix} buildable: false -""".format( - a_prefix=str(fake_prefix) - ) +""".format(a_prefix=str(fake_prefix)) ) external_config_dict = spack.util.spack_yaml.load_config(external_config) @@ -1226,7 +1231,9 @@ def test_env_with_config(environment_from_manifest): with e: e.concretize() - assert any(x.intersects("mpileaks@2.2") for x in e._get_environment_specs()) + mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec("mpileaks")) + mpileaks = e.specs_by_hash[mpileaks_hash] + assert mpileaks.satisfies("mpileaks@2.2") def test_with_config_bad_include_create(environment_from_manifest): @@ -1310,10 +1317,13 @@ def test_env_with_include_config_files_same_basename( with e: e.concretize() - environment_specs = e._get_environment_specs(False) + mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec("mpileaks")) + mpileaks = e.specs_by_hash[mpileaks_hash] + assert mpileaks.satisfies("mpileaks@2.2") - assert environment_specs[0].satisfies("libelf@0.8.10") - assert environment_specs[1].satisfies("mpileaks@2.2") + libelf_hash = next(x.hash for x in e.concretized_roots if x.root == Spec("libelf")) + libelf = e.specs_by_hash[libelf_hash] + assert libelf.satisfies("libelf@0.8.10") @pytest.fixture(scope="function") @@ -1340,9 +1350,7 @@ def mpileaks_env_config(include_path): - {0} specs: - mpileaks -""".format( - include_path - ) +""".format(include_path) def test_env_with_included_config_file(mutable_mock_env_path, packages_file): @@ -1370,7 +1378,9 @@ def test_env_with_included_config_file(mutable_mock_env_path, packages_file): with e: e.concretize() - assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs()) + mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec("mpileaks")) + mpileaks = e.specs_by_hash[mpileaks_hash] + assert mpileaks.satisfies("mpileaks@2.2") def test_config_change_existing( @@ -1534,7 +1544,9 @@ def test_env_with_included_config_scope(mutable_mock_env_path, packages_file): with e: e.concretize() - assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs()) + mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec("mpileaks")) + mpileaks = e.specs_by_hash[mpileaks_hash] + assert mpileaks.satisfies("mpileaks@2.2") def test_env_with_included_config_var_path(tmp_path: pathlib.Path, packages_file): @@ -1555,7 +1567,9 @@ def test_env_with_included_config_var_path(tmp_path: pathlib.Path, packages_file with e: e.concretize() - assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs()) + mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec("mpileaks")) + mpileaks = e.specs_by_hash[mpileaks_hash] + assert mpileaks.satisfies("mpileaks@2.2") def test_env_with_included_config_precedence(tmp_path: pathlib.Path): @@ -1592,13 +1606,15 @@ def test_env_with_included_config_precedence(tmp_path: pathlib.Path): e = ev.Environment(tmp_path) with e: e.concretize() - specs = e._get_environment_specs() + + mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec("mpileaks")) + mpileaks = e.specs_by_hash[mpileaks_hash] # ensure included scope took effect - assert any(x.satisfies("mpileaks@2.2") for x in specs) + assert mpileaks.satisfies("mpileaks@2.2") # ensure env file takes precedence - assert any(x.satisfies("libelf@0.8.12") for x in specs) + assert mpileaks["libelf"].satisfies("libelf@0.8.12") def test_env_with_included_configs_precedence(tmp_path: pathlib.Path): @@ -1641,13 +1657,15 @@ def test_env_with_included_configs_precedence(tmp_path: pathlib.Path): e = ev.Environment(tmp_path) with e: e.concretize() - specs = e._get_environment_specs() - # ensure included package spec took precedence over manifest spec - assert any(x.satisfies("mpileaks@2.2") for x in specs) + mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec("mpileaks")) + mpileaks = e.specs_by_hash[mpileaks_hash] + + # ensure the included package spec took precedence over manifest spec + assert mpileaks.satisfies("mpileaks@2.2") - # ensure first included package spec took precedence over one from second - assert any(x.satisfies("libelf@0.8.10") for x in specs) + # ensure the first included package spec took precedence over one from second + assert mpileaks["libelf"].satisfies("libelf@0.8.10") @pytest.mark.regression("39248") @@ -1847,15 +1865,15 @@ def test_uninstall_keeps_in_env(mock_stage, mock_fetch, install_mockery): test = ev.read("test") # Save this spec to check later if it is still in the env (mpileaks_hash,) = list(x for x, y in test.specs_by_hash.items() if y.name == "mpileaks") - orig_user_specs = test.user_specs - orig_concretized_specs = test.concretized_order + user_specs_before = test.user_specs + user_spec_hashes_before = {x.hash for x in test.concretized_roots} with ev.read("test"): uninstall("-ya") test = ev.read("test") - assert test.concretized_order == orig_concretized_specs - assert test.user_specs.specs == orig_user_specs.specs + assert {x.hash for x in test.concretized_roots} == user_spec_hashes_before + assert test.user_specs.specs == user_specs_before.specs assert mpileaks_hash in test.specs_by_hash assert not test.specs_by_hash[mpileaks_hash].installed @@ -1873,7 +1891,7 @@ def test_uninstall_removes_from_env(mock_stage, mock_fetch, install_mockery): test = ev.read("test") assert not test.specs_by_hash - assert not test.concretized_order + assert not test.concretized_roots assert not test.user_specs @@ -1897,8 +1915,8 @@ def test_indirect_build_dep(repo_builder: RepoBuilder): e.write() e_read = ev.read("test") - (x_env_hash,) = e_read.concretized_order - + assert len(e_read.concretized_roots) == 1 + x_env_hash = e_read.concretized_roots[0].hash x_env_spec = e_read.specs_by_hash[x_env_hash] assert x_env_spec == x_concretized @@ -1939,7 +1957,7 @@ def test_store_different_build_deps(repo_builder: RepoBuilder): e.write() e_read = ev.read("test") - y_env_hash, x_env_hash = e_read.concretized_order + y_env_hash, x_env_hash = [x.hash for x in e_read.concretized_roots] y_read = e_read.specs_by_hash[y_env_hash] x_read = e_read.specs_by_hash[x_env_hash] @@ -2051,8 +2069,8 @@ def test_env_include_concrete_env_yaml(env_name): combined = ev.read("combined_env") combined_yaml = combined.manifest["spack"] - assert "include_concrete" in combined_yaml - assert test.path in combined_yaml["include_concrete"] + assert ev.lockfile_include_key in combined_yaml + assert test.path in combined_yaml[ev.lockfile_include_key] @pytest.mark.regression("45766") @@ -2087,8 +2105,8 @@ def test_env_multiple_include_concrete_envs(): combined_yaml = combined.manifest["spack"] - assert test1.path in combined_yaml["include_concrete"][0] - assert test2.path in combined_yaml["include_concrete"][1] + assert test1.path in combined_yaml[ev.lockfile_include_key][0] + assert test2.path in combined_yaml[ev.lockfile_include_key][1] # No local specs in the combined env assert not combined_yaml["specs"] @@ -2099,17 +2117,17 @@ def test_env_include_concrete_envs_lockfile(): combined_yaml = combined.manifest["spack"] - assert "include_concrete" in combined_yaml - assert test1.path in combined_yaml["include_concrete"] + assert ev.lockfile_include_key in combined_yaml + assert test1.path in combined_yaml[ev.lockfile_include_key] with open(combined.lock_path, encoding="utf-8") as f: lockfile_as_dict = combined._read_lockfile(f) assert set( - entry["hash"] for entry in lockfile_as_dict["include_concrete"][test1.path]["roots"] + entry["hash"] for entry in lockfile_as_dict[ev.lockfile_include_key][test1.path]["roots"] ) == set(test1.specs_by_hash) assert set( - entry["hash"] for entry in lockfile_as_dict["include_concrete"][test2.path]["roots"] + entry["hash"] for entry in lockfile_as_dict[ev.lockfile_include_key][test2.path]["roots"] ) == set(test2.specs_by_hash) @@ -2126,13 +2144,13 @@ def test_env_include_concrete_add_env(): new_env.write() # add new env to combined - combined.included_concrete_envs.append(new_env.path) + combined.included_concrete_env_root_dirs.append(new_env.path) # assert thing haven't changed yet with open(combined.lock_path, encoding="utf-8") as f: lockfile_as_dict = combined._read_lockfile(f) - assert new_env.path not in lockfile_as_dict["include_concrete"].keys() + assert new_env.path not in lockfile_as_dict[ev.lockfile_include_key].keys() # concretize combined env with new env combined.concretize() @@ -2142,20 +2160,20 @@ def test_env_include_concrete_add_env(): with open(combined.lock_path, encoding="utf-8") as f: lockfile_as_dict = combined._read_lockfile(f) - assert new_env.path in lockfile_as_dict["include_concrete"].keys() + assert new_env.path in lockfile_as_dict[ev.lockfile_include_key].keys() def test_env_include_concrete_remove_env(): test1, test2, combined = setup_combined_multiple_env() # remove test2 from combined - combined.included_concrete_envs = [test1.path] + combined.included_concrete_env_root_dirs = [test1.path] # assert test2 is still in combined's lockfile with open(combined.lock_path, encoding="utf-8") as f: lockfile_as_dict = combined._read_lockfile(f) - assert test2.path in lockfile_as_dict["include_concrete"].keys() + assert test2.path in lockfile_as_dict[ev.lockfile_include_key].keys() # reconcretize combined combined.concretize() @@ -2165,7 +2183,7 @@ def test_env_include_concrete_remove_env(): with open(combined.lock_path, encoding="utf-8") as f: lockfile_as_dict = combined._read_lockfile(f) - assert test2.path not in lockfile_as_dict["include_concrete"].keys() + assert test2.path not in lockfile_as_dict[ev.lockfile_include_key].keys() def configure_reuse(reuse_mode, combined_env) -> Optional[ev.Environment]: @@ -2220,7 +2238,7 @@ def configure_reuse(reuse_mode, combined_env) -> Optional[ev.Environment]: "from_environment_raise", ], ) -def test_env_include_concrete_reuse(do_not_check_runtimes_on_reuse, reuse_mode): +def test_env_include_concrete_reuse(reuse_mode): # The default mpi version is 3.x provided by mpich in the mock repo. # This test verifies that concretizing with an included concrete # environment with "concretizer:reuse:true" the included @@ -2271,22 +2289,22 @@ def test_env_include_concrete_reuse(do_not_check_runtimes_on_reuse, reuse_mode): @pytest.mark.parametrize("unify", [True, False, "when_possible"]) -def test_env_include_concrete_env_reconcretized(unify): +def test_env_include_concrete_env_reconcretized(mutable_config, unify): """Double check to make sure that concrete_specs for the local specs is empty after reconcretizing. """ _, _, combined = setup_combined_multiple_env() - combined.unify = unify - with open(combined.lock_path, encoding="utf-8") as f: lockfile_as_dict = combined._read_lockfile(f) assert not lockfile_as_dict["roots"] assert not lockfile_as_dict["concrete_specs"] - combined.concretize() - combined.write() + with combined: + mutable_config.set("concretizer:unify", unify) + combined.concretize() + combined.write() with open(combined.lock_path, encoding="utf-8") as f: lockfile_as_dict = combined._read_lockfile(f) @@ -2296,20 +2314,27 @@ def test_env_include_concrete_env_reconcretized(unify): def test_concretize_include_concrete_env(): + """Tests that if we update an included environment, and later we re-concretize the environment + that includes it, we use the latest version of the concrete specs. + """ test1, _, combined = setup_combined_multiple_env() + # Update test1 environment with test1: add("mpileaks") test1.concretize() test1.write() - assert Spec("mpileaks") in test1.concretized_user_specs - assert Spec("mpileaks") not in combined.included_concretized_user_specs[test1.path] + # Check the test1 environment includes mpileaks, while the combined environment does not + assert Spec("mpileaks") in {x.root for x in test1.concretized_roots} + assert Spec("mpileaks") not in { + x.root for x in combined.included_concretized_roots[test1.path] + } + # If we update the combined environment, it will include mpileaks too combined.concretize() combined.write() - - assert Spec("mpileaks") in combined.included_concretized_user_specs[test1.path] + assert Spec("mpileaks") in {x.root for x in combined.included_concretized_roots[test1.path]} def test_concretize_nested_include_concrete_envs(): @@ -2333,10 +2358,13 @@ def test_concretize_nested_include_concrete_envs(): with open(test3.lock_path, encoding="utf-8") as f: lockfile_as_dict = test3._read_lockfile(f) - assert test2.path in lockfile_as_dict["include_concrete"] - assert test1.path in lockfile_as_dict["include_concrete"][test2.path]["include_concrete"] + assert test2.path in lockfile_as_dict[ev.lockfile_include_key] + assert ( + test1.path + in lockfile_as_dict[ev.lockfile_include_key][test2.path][ev.lockfile_include_key] + ) - assert Spec("zlib") in test3.included_concretized_user_specs[test1.path] + assert Spec("zlib") in {x.root for x in test3.included_concretized_roots[test1.path]} def test_concretize_nested_included_concrete(): @@ -2357,7 +2385,7 @@ def test_concretize_nested_included_concrete(): test2.concretize() test2.write() - assert Spec("zlib") in test2.included_concretized_user_specs[test1.path] + assert Spec("zlib") in {x.root for x in test2.included_concretized_roots[test1.path]} # Modify/re-concretize test1 to replace zlib with mpileaks with test1: @@ -2374,9 +2402,9 @@ def test_concretize_nested_included_concrete(): test3.concretize() test3.write() - included_specs = test3.included_concretized_user_specs[test1.path] - assert len(included_specs) == 1 - assert Spec("mpileaks") in included_specs + included_roots = test3.included_concretized_roots[test1.path] + assert len(included_roots) == 1 + assert Spec("mpileaks") in {x.root for x in included_roots} # The last concretization of test4's included environments should have test2 # with the original concretized test1 spec and test3 with the re-concretized @@ -2386,7 +2414,7 @@ def test_concretize_nested_included_concrete(): def included_included_spec(path1, path2): included_path1 = test4.included_concrete_spec_data[path1] - included_path2 = included_path1["include_concrete"][path2] + included_path2 = included_path1[ev.lockfile_include_key][path2] return included_path2["roots"][0]["spec"] included_test2_test1 = included_included_spec(test2.path, test1.path) @@ -2755,18 +2783,18 @@ def test_stack_yaml_force_remove_from_matrix(tmp_path: pathlib.Path): e.concretize() before_user = e.user_specs.specs - before_conc = e.concretized_user_specs + concretized_roots_before = e.concretized_roots remove("-f", "-l", "packages", "mpileaks") after_user = e.user_specs.specs - after_conc = e.concretized_user_specs + concretized_roots_after = e.concretized_roots assert before_user == after_user mpileaks_spec = Spec("mpileaks target=default_target") - assert mpileaks_spec in before_conc - assert mpileaks_spec not in after_conc + assert mpileaks_spec in {x.root for x in concretized_roots_before} + assert mpileaks_spec not in {x.root for x in concretized_roots_after} def test_stack_definition_extension(tmp_path: pathlib.Path): @@ -2970,7 +2998,7 @@ def test_stack_combinatorial_view( """Tests creating a default view for a combinatorial stack.""" view_dir = tmp_path / "view" with installed_environment(template_combinatorial_env.format(view_config="")) as test: - for spec in test._get_environment_specs(): + for spec in traverse_nodes(test.concrete_roots(), deptype=("link", "run")): if spec.name == "gcc-runtime": continue current_dir = view_dir / f"{spec.architecture.target}" / f"{spec.name}-{spec.version}" @@ -2983,7 +3011,7 @@ def test_stack_view_select( view_dir = tmp_path / "view" content = template_combinatorial_env.format(view_config="select: ['target=x86_64']\n") with installed_environment(content) as test: - for spec in test._get_environment_specs(): + for spec in traverse_nodes(test.concrete_roots(), deptype=("link", "run")): if spec.name == "gcc-runtime": continue current_dir = view_dir / f"{spec.architecture.target}" / f"{spec.name}-{spec.version}" @@ -2996,7 +3024,7 @@ def test_stack_view_exclude( view_dir = tmp_path / "view" content = template_combinatorial_env.format(view_config="exclude: [callpath]\n") with installed_environment(content) as test: - for spec in test._get_environment_specs(): + for spec in traverse_nodes(test.concrete_roots(), deptype=("link", "run")): if spec.name == "gcc-runtime": continue current_dir = view_dir / f"{spec.architecture.target}" / f"{spec.name}-{spec.version}" @@ -3013,7 +3041,7 @@ def test_stack_view_select_and_exclude( """ ) with installed_environment(content) as test: - for spec in test._get_environment_specs(): + for spec in traverse_nodes(test.concrete_roots(), deptype=("link", "run")): if spec.name == "gcc-runtime": continue current_dir = view_dir / f"{spec.architecture.target}" / f"{spec.name}-{spec.version}" @@ -3033,7 +3061,7 @@ def test_view_link_roots( """ ) with installed_environment(content) as test: - for spec in test._get_environment_specs(): + for spec in traverse_nodes(test.concrete_roots(), deptype=("link", "run")): if spec.name == "gcc-runtime": continue current_dir = view_dir / f"{spec.architecture.target}" / f"{spec.name}-{spec.version}" @@ -3078,7 +3106,7 @@ def test_view_link_run( "dtlink2", "dtlink3", "dtlink4", - "dtlink5" "dtbuild1", + "dtlink5dtbuild1", "dtbuild2", "dtbuild3", ): @@ -3115,7 +3143,7 @@ def test_view_link_all(installed_environment, template_combinatorial_env, tmp_pa ) with installed_environment(content) as test: - for spec in test._get_environment_specs(): + for spec in traverse_nodes(test.concrete_roots(), deptype=("link", "run")): if spec.name == "gcc-runtime": continue current_dir = view_dir / f"{spec.architecture.target}" / f"{spec.name}-{spec.version}" @@ -3254,7 +3282,7 @@ def test_stack_view_multiple_views(installed_environment, tmp_path: pathlib.Path with installed_environment(content) as e: assert os.path.exists(str(default_dir / "bin")) - for spec in e._get_environment_specs(): + for spec in traverse_nodes(e.concrete_roots(), deptype=("link", "run")): if spec.name == "gcc-runtime": continue current_dir = comb_dir / f"{spec.architecture.target}" / f"{spec.name}-{spec.version}" @@ -3335,53 +3363,52 @@ def test_env_activate_custom_view(tmp_path: pathlib.Path, mock_packages): assert os.path.join(nondefaultdir, "bin") in shell -def test_concretize_user_specs_together(): - e = ev.create("coconcretization") - e.unify = True +def test_concretize_user_specs_together(mutable_config): + with ev.create("coconcretization") as e: + mutable_config.set("concretizer:unify", True) - # Concretize a first time using 'mpich' as the MPI provider - e.add("mpileaks") - e.add("mpich") - e.concretize() - - assert all("mpich" in spec for _, spec in e.concretized_specs()) - assert all("mpich2" not in spec for _, spec in e.concretized_specs()) + # Concretize a first time using 'mpich' as the MPI provider + e.add("mpileaks") + e.add("mpich") + e.concretize() - # Concretize a second time using 'mpich2' as the MPI provider - e.remove("mpich") - e.add("mpich2") + assert all("mpich" in spec for _, spec in e.concretized_specs()) + assert all("mpich2" not in spec for _, spec in e.concretized_specs()) - exc_cls = spack.error.UnsatisfiableSpecError + # Concretize a second time using 'mpich2' as the MPI provider + e.remove("mpich") + e.add("mpich2") - # Concretizing without invalidating the concrete spec for mpileaks fails - with pytest.raises(exc_cls): - e.concretize() - e.concretize(force=True) + exc_cls = spack.error.UnsatisfiableSpecError - assert all("mpich2" in spec for _, spec in e.concretized_specs()) - assert all("mpich" not in spec for _, spec in e.concretized_specs()) + # Concretizing without invalidating the concrete spec for mpileaks fails + with pytest.raises(exc_cls): + e.concretize() + e.concretize(force=True) - # Concretize again without changing anything, check everything - # stays the same - e.concretize() + assert all("mpich2" in spec for _, spec in e.concretized_specs()) + assert all("mpich" not in spec for _, spec in e.concretized_specs()) - assert all("mpich2" in spec for _, spec in e.concretized_specs()) - assert all("mpich" not in spec for _, spec in e.concretized_specs()) + # Concretize again without changing anything, check everything + # stays the same + e.concretize() + assert all("mpich2" in spec for _, spec in e.concretized_specs()) + assert all("mpich" not in spec for _, spec in e.concretized_specs()) -def test_duplicate_packages_raise_when_concretizing_together(): - e = ev.create("coconcretization") - e.unify = True - e.add("mpileaks+opt") - e.add("mpileaks~opt") - e.add("mpich") +def test_duplicate_packages_raise_when_concretizing_together(mutable_config): + with ev.create("coconcretization") as e: + mutable_config.set("concretizer:unify", True) + e.add("mpileaks+opt") + e.add("mpileaks~opt") + e.add("mpich") - exc_cls = spack.error.UnsatisfiableSpecError - match = r"You could consider setting `concretizer:unify`" + exc_cls = spack.error.UnsatisfiableSpecError + match = r"You could consider setting `concretizer:unify`" - with pytest.raises(exc_cls, match=match): - e.concretize() + with pytest.raises(exc_cls, match=match): + e.concretize() def test_env_write_only_non_default(): @@ -3510,9 +3537,7 @@ def test_lockfile_not_deleted_on_write_error(tmp_path: pathlib.Path, monkeypatch def _write_helper_raise(self): raise RuntimeError("some error") - monkeypatch.setattr( - spack.environment.environment.EnvironmentManifestFile, "flush", _write_helper_raise - ) + monkeypatch.setattr(ev.environment.EnvironmentManifestFile, "flush", _write_helper_raise) with ev.Environment(str(tmp_path)) as e: e.concretize(force=True) with pytest.raises(RuntimeError): @@ -3581,18 +3606,16 @@ def test_does_not_rewrite_rel_dev_path_when_keep_relative_is_set(tmp_path: pathl @pytest.mark.regression("23440") -def test_custom_version_concretize_together(): +def test_custom_version_concretize_together(mutable_config): # Custom versions should be permitted in specs when # concretizing together - e = ev.create("custom_version") - e.unify = True - - # Concretize a first time using 'mpich' as the MPI provider - e.add("hdf5@=myversion") - e.add("mpich") - e.concretize() - - assert any(spec.satisfies("hdf5@myversion") for _, spec in e.concretized_specs()) + with ev.create("custom_version") as e: + mutable_config.set("concretizer:unify", True) + # Concretize a first time using 'mpich' as the MPI provider + e.add("hdf5@=myversion") + e.add("mpich") + e.concretize() + assert any(spec.satisfies("hdf5@myversion") for _, spec in e.concretized_specs()) def test_modules_relative_to_views(environment_from_manifest, install_mockery, mock_fetch): @@ -3612,8 +3635,8 @@ def test_modules_relative_to_views(environment_from_manifest, install_mockery, m with ev.read("test") as e: install("--fake") - - spec = e.specs_by_hash[e.concretized_order[0]] + user_spec_hash = e.concretized_roots[0].hash + spec = e.specs_by_hash[user_spec_hash] view_prefix = e.default_view.get_projection_for_spec(spec) modules_glob = "%s/modules/**/*/*" % e.path modules = glob.glob(modules_glob) @@ -3708,15 +3731,13 @@ def _always_fail(cls, *args, **kwargs): @pytest.mark.regression("24148") -def test_virtual_spec_concretize_together(): +def test_virtual_spec_concretize_together(mutable_config): # An environment should permit to concretize "mpi" - e = ev.create("virtual_spec") - e.unify = True - - e.add("mpi") - e.concretize() - - assert any(s.package.provides("mpi") for _, s in e.concretized_specs()) + with ev.create("virtual_spec") as e: + mutable_config.set("concretizer:unify", True) + e.add("mpi") + e.concretize() + assert any(s.package.provides("mpi") for _, s in e.concretized_specs()) def test_query_develop_specs(tmp_path: pathlib.Path): @@ -3761,9 +3782,7 @@ def test_custom_store_in_environment(mutable_config, tmp_path: pathlib.Path): config: install_tree: root: {0} -""".format( - install_root - ) +""".format(install_root) ) current_store_root = str(spack.store.STORE.root) assert str(current_store_root) != str(install_root) @@ -4346,30 +4365,22 @@ def test_depfile_empty_does_not_error(tmp_path: pathlib.Path): assert make.returncode == 0 -def test_unify_when_possible_works_around_conflicts(): - e = ev.create("coconcretization") - e.unify = "when_possible" - - e.add("mpileaks+opt") - e.add("mpileaks~opt") - e.add("mpich") - - e.concretize() +def test_unify_when_possible_works_around_conflicts(mutable_config): + with ev.create("coconcretization") as e: + mutable_config.set("concretizer:unify", "when_possible") + e.add("mpileaks+opt") + e.add("mpileaks~opt") + e.add("mpich") + e.concretize() - assert len([x for x in e.all_specs() if x.satisfies("mpileaks")]) == 2 - assert len([x for x in e.all_specs() if x.satisfies("mpileaks+opt")]) == 1 - assert len([x for x in e.all_specs() if x.satisfies("mpileaks~opt")]) == 1 - assert len([x for x in e.all_specs() if x.satisfies("mpich")]) == 1 + assert len([x for x in e.all_specs() if x.satisfies("mpileaks")]) == 2 + assert len([x for x in e.all_specs() if x.satisfies("mpileaks+opt")]) == 1 + assert len([x for x in e.all_specs() if x.satisfies("mpileaks~opt")]) == 1 + assert len([x for x in e.all_specs() if x.satisfies("mpich")]) == 1 -# Using mock_include_cache to ensure the "remote" file is cached in a temporary -# location and not polluting the user cache. def test_env_include_packages_url( - tmp_path: pathlib.Path, - mutable_empty_config, - mock_fetch_url_text, - mock_curl_configs, - mock_include_cache, + tmp_path: pathlib.Path, mutable_empty_config, mock_fetch_url_text, mock_curl_configs ): """Test inclusion of a (GitHub) URL.""" develop_url = "https://github.com/fake/fake/blob/develop/" @@ -4496,7 +4507,7 @@ def test_env_include_mixed_views( f"""\ spack: include: -{''.join(includes)} +{"".join(includes)} specs: - mpileaks """ @@ -4564,7 +4575,7 @@ def test_stack_view_multiple_views_same_name( # the view root in the included view should NOT exist assert not os.path.exists(str(default_dir)) - for spec in e._get_environment_specs(): + for spec in traverse_nodes(e.concrete_roots(), deptype=("link", "run")): # no specs will exist in the included view projection base_dir = view_dir / f"{spec.architecture.target}" included_dir = base_dir / f"{spec.name}-{spec.version}-from-view" @@ -4635,3 +4646,351 @@ def test_non_str_repos(installed_environment): branch: develop""" ): pass + + +def test_concretized_specs_and_include_concrete(mutable_config): + """Tests the consistency of concretized specs when there are either + duplicate input specs or duplicate hashes. + """ + # Create a structure like this one + # + # Local specs: + # - mpileaks -> hash1 + # - libelf@0.8.12 -> hash2 + # - pkg-a -> hash3 + # + # Included specs: + # - mpileaks -> hash4 + # - libelf -> hash2 + # - pkg-a -> hash3 + env("create", "included-env") + with ev.read("included-env") as e: + e.add("mpileaks") + e.add("libelf") + e.add("pkg-a") + mutable_config.set( + "packages", {"mpileaks": {"require": ["@2.2"]}, "libelf": {"require": ["@0.8.12"]}} + ) + included_pairs = e.concretize() + e.write() + + env("create", "--include-concrete", "included-env", "main-env") + with ev.read("main-env") as e: + e.add("mpileaks") + e.add("libelf@0.8.12") + e.add("pkg-a") + mutable_config.set("packages", {"mpileaks": {"require": ["@2.3"]}}) + spec_pairs = e.concretize() + concretized_specs = list(e.concretized_specs()) + assert list(dedupe(spec_pairs + included_pairs)) == concretized_specs + assert len(concretized_specs) == 5 + + +def test_view_can_select_group_of_specs(installed_environment, tmp_path: pathlib.Path): + """Tests that we can select groups of specs in a view and exclude other groups""" + view_dir = tmp_path / "view" + with installed_environment( + f"""\ +spack: + specs: + - group: apps1 + specs: + - mpileaks + - group: apps2 + specs: + - cmake + - group: apps3 + specs: + - pkg-a + view: + default: + root: {view_dir} + group: [apps1, apps2] +""" + ) as test: + for item in test.concretized_roots: + # Assertions are based on the behavior of the "--fake" install + bin_file = pathlib.Path(test.default_view.view()._root) / "bin" / item.root.name + assert not bin_file.exists() if item.group == "apps3" else bin_file.exists() + + +def test_view_can_select_group_of_specs_using_string( + installed_environment, tmp_path: pathlib.Path +): + """Tests that we can select groups of specs in a view and exclude other groups""" + view_dir = tmp_path / "view" + with installed_environment( + f"""\ +spack: + specs: + - group: apps1 + specs: + - mpileaks + - group: apps2 + specs: + - cmake + view: + default: + root: {view_dir} + group: apps1 +""" + ) as test: + for item in test.concretized_roots: + # Assertions are based on the behavior of the "--fake" install + bin_file = pathlib.Path(test.default_view.view()._root) / "bin" / item.root.name + assert not bin_file.exists() if item.group == "apps2" else bin_file.exists() + + +def test_env_include_concrete_only(tmp_path, mock_packages, mutable_config): + """Confirm that an environment that only includes a concrete environment actually loads it.""" + specs = ["libdwarf", "libelf"] + + include_dir = tmp_path / "includes" + include_dir.mkdir() + include_manifest = include_dir / ev.manifest_name + include_manifest.write_text( + f"""\ +spack: + specs: + - {specs[0]} + - {specs[1]} +""" + ) + include_env = ev.create("test_include", include_manifest) + include_env.concretize() + include_env.write() + + include_lockfile = include_env.lock_path + assert os.path.exists(include_lockfile) + + manifest_file = tmp_path / ev.manifest_name + manifest_file.write_text( + f"""\ +spack: + include: + - {str(include_lockfile)} +""" + ) + e = ev.create("test", manifest_file) + + # Confirm the only specs the environment has are those loaded from the + # lockfile. + assert len(e.user_specs) == 0 + all_concrete = [s for s, _ in e.concretized_specs()] + for spec in specs: + assert Spec(spec) in all_concrete + + +@pytest.mark.parametrize( + "concrete,includes", + [ + (["$HOME/path/to/other/environment"], []), + (["$HOME/path/to/another/environment"], ["a/b", "$HOME/includes"]), + ], +) +def test_env_update_include_concrete(tmp_path: pathlib.Path, concrete, includes): + """Confirm update of include_concrete converts it to include.""" + + config = {"include_concrete": concrete} + if includes: + config["include"] = includes + new_concrete = [os.path.join(p, ev.lockfile_name) for p in concrete] + assert spack.schema.env.update(config) + assert "include_concrete" not in config + assert config["include"] == new_concrete + includes + + +def test_include_concrete_deprecation_warning( + tmp_path: pathlib.Path, environment_from_manifest, capfd +): + try: + environment_from_manifest( + """\ +spack: + include_concrete: + - /path/to/some/environment +""" + ) + except ev.SpackEnvironmentError: + pass + + _, err = capfd.readouterr() + assert "should be 'include'" in err + + +def test_env_include_concrete_relative_path(tmp_path, mock_packages, mutable_config): + """Tests that a relative path in 'include' for a spack.lock is resolved relative to the + manifest file, not the current working directory. + """ + # Create and concretize the included environment. + include_dir = tmp_path / "include_env" + include_dir.mkdir() + (include_dir / ev.manifest_name).write_text( + """\ +spack: + specs: + - libdwarf +""" + ) + with ev.Environment(str(include_dir)) as e: + e.concretize() + e.write() + assert os.path.exists(e.lock_path) + + # Create the main environment in a sibling directory, using a *relative* path + main_dir = tmp_path / "main_env" + main_dir.mkdir() + relative_lockfile = f"../include_env/{ev.lockfile_name}" + (main_dir / ev.manifest_name).write_text( + f"""\ +spack: + include: + - {relative_lockfile} +""" + ) + with ev.Environment(str(main_dir)) as e: + e.concretize() + e.write() + assert len(e.user_specs) == 0 + assert [s for s, _ in e.concretized_specs()] == [Spec("libdwarf")] + + +def test_env_include_concrete_git_lockfile(tmp_path, mock_packages, mutable_config, monkeypatch): + """Tests that a spack.lock listed inside a git-based include is resolved using the + clone destination as the base, not the manifest directory. + """ + # Create and concretize the included environment. + include_dir = tmp_path / "include_env" + include_dir.mkdir() + (include_dir / ev.manifest_name).write_text( + """\ +spack: + specs: + - libdwarf +""" + ) + with ev.Environment(str(include_dir)) as e: + e.concretize() + e.write() + assert os.path.exists(e.lock_path) + + # Simulate a cloned git repo: the spack.lock lives at a subpath within the clone. + clone_dest = tmp_path / "git_clone" + lock_subpath = "envs/staging/spack.lock" + lock_in_clone = clone_dest / "envs" / "staging" / ev.lockfile_name + lock_in_clone.parent.mkdir(parents=True) + shutil.copy(e.lock_path, lock_in_clone) + # is_env_dir() requires spack.yaml alongside spack.lock + shutil.copy(os.path.join(e.path, ev.manifest_name), lock_in_clone.parent) + + # Prevent actual git operations; return the pre-built clone destination. + monkeypatch.setattr( + spack.config.GitIncludePaths, "_clone", lambda self, parent_scope: str(clone_dest) + ) + + main_dir = tmp_path / "main_env" + main_dir.mkdir() + (main_dir / ev.manifest_name).write_text( + f"""\ +spack: + include: + - git: https://example.com/configs.git + branch: main + paths: + - {lock_subpath} +""" + ) + with ev.Environment(str(main_dir)) as e: + e.concretize() + e.write() + assert len(e.user_specs) == 0 + assert [s for s, _ in e.concretized_specs()] == [Spec("libdwarf")] + + +@pytest.mark.skipif(sys.platform != "linux", reason="Target is linux-specific") +def test_compiler_target_env(mock_packages, environment_from_manifest): + """Tests that Spack doesn't drop flag definitions on compilers + when a target is required in config. + """ + + cflags = "-Wall" + env = environment_from_manifest( + f"""\ +spack: + specs: + - libdwarf %c=gcc@12.100.100 + packages: + all: + require: + - "target=x86_64_v3" + gcc: + externals: + - spec: gcc@12.100.100 languages:=c,c++ + prefix: /fake + extra_attributes: + compilers: + c: /fake/bin/gcc + cxx: /fake/bin/g++ + flags: + cflags: {cflags} + require: "gcc" +""" + ) + + with env: + env.concretize() + libdwarf = env.concrete_roots()[0] + assert libdwarf.satisfies("cflags=-Wall") + # Sanity check: make sure the target we expect was applied to the + # compiler entry + assert libdwarf["c"].satisfies("gcc@12.100.100 languages:=c,c++ target=x86_64_v3") + + +@pytest.mark.regression("52247") +def test_create_with_orphaned_directory(mutable_mock_env_path: pathlib.Path): + """Tests that an orphaned environment directory (directory exists, no spack.yaml) must not + prevent 'spack env create' from creating a new environment with that name. + """ + orphaned = mutable_mock_env_path / "test1" + orphaned_subdir = orphaned / ".spack-env" + orphaned_subdir.mkdir(parents=True) + + # The orphaned directory must not be seen as an existing environment + assert not ev.exists("test1") + + # Creating an environment over an orphaned directory must succeed + env("create", "test1") + + assert ev.exists("test1") + assert "test1" in env("list") + + +@pytest.mark.parametrize( + "setup", + [ + # valid environment: spack.yaml is a regular file + pytest.param("valid", id="valid"), + # orphaned directory: no spack.yaml at all + pytest.param("orphaned", id="orphaned"), + # broken manifest symlink: spack.yaml points to a non-existent target + pytest.param("broken_symlink", id="broken_symlink"), + ], +) +@pytest.mark.regression("52247") +def test_exists_consistent_with_all_environment_names( + mutable_mock_env_path: pathlib.Path, setup: str +): + """Tests that exists() and all_environment_names() agree on whether an environment exists.""" + env_dir = mutable_mock_env_path / "myenv" + env_dir.mkdir(parents=True) + manifest = env_dir / ev.manifest_name + + if setup == "valid": + manifest.write_text(ev.default_manifest_yaml()) + elif setup == "orphaned": + pass # no manifest + elif setup == "broken_symlink": + manifest.symlink_to("/nonexistent/spack.yaml") + + listed = "myenv" in ev.all_environment_names() + assert ev.exists("myenv") == listed diff --git a/lib/spack/spack/test/cmd/external.py b/lib/spack/spack/test/cmd/external.py index 8c0d06d3bc192a..57f6d0b1e2ee79 100644 --- a/lib/spack/spack/test/cmd/external.py +++ b/lib/spack/spack/test/cmd/external.py @@ -321,16 +321,6 @@ def test_failures_in_scanning_do_not_result_in_an_error( mock_executable, monkeypatch, mutable_config ): """Tests that scanning paths with wrong permissions, won't cause `external find` to error.""" - versions = {"first": "3.19.1", "second": "3.23.3"} - - @classmethod - def _determine_version(cls, exe): - bin_parent = os.path.dirname(exe).split(os.sep)[-2] - return versions[bin_parent] - - cmake_cls = spack.repo.PATH.get_pkg_class("cmake") - monkeypatch.setattr(cmake_cls, "determine_version", _determine_version) - cmake_exe1 = mock_executable( "cmake", output="echo cmake version 3.19.1", subdir=("first", "bin") ) @@ -338,18 +328,31 @@ def _determine_version(cls, exe): "cmake", output="echo cmake version 3.23.3", subdir=("second", "bin") ) - # Remove access from the first directory executable - cmake_exe1.parent.chmod(0o600) + @classmethod + def _determine_version(cls, exe): + name = pathlib.Path(exe).parent.parent.name + if name == "first": + return "3.19.1" + elif name == "second": + return "3.23.3" + assert False, f"Unexpected exe path {exe}" - value = os.pathsep.join([str(cmake_exe1.parent), str(cmake_exe2.parent)]) - monkeypatch.setenv("PATH", value) + cmake_cls = spack.repo.PATH.get_pkg_class("cmake") + monkeypatch.setattr(cmake_cls, "determine_version", _determine_version) + monkeypatch.setenv("PATH", f"{cmake_exe1.parent}{os.pathsep}{cmake_exe2.parent}") + + try: + # Remove access from the first directory executable + cmake_exe1.parent.chmod(0o600) + output = external("find", "cmake") + finally: + cmake_exe1.parent.chmod(0o700) - output = external("find", "cmake") assert external.returncode == 0 assert "The following specs have been" in output assert "cmake" in output - for vers in versions.values(): - assert vers in output + assert "3.19.1" in output + assert "3.23.3" in output def test_detect_virtuals(mock_executable, mutable_config, monkeypatch): diff --git a/lib/spack/spack/test/cmd/find.py b/lib/spack/spack/test/cmd/find.py index aa5ac1ee676e91..aaa8adba0c1ff4 100644 --- a/lib/spack/spack/test/cmd/find.py +++ b/lib/spack/spack/test/cmd/find.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse +import io import json import os import pathlib @@ -16,6 +17,7 @@ import spack.package_base import spack.paths import spack.repo +import spack.spec import spack.store import spack.user_environment as uenv from spack.enums import InstallRecordStatus @@ -215,6 +217,15 @@ def test_display_json_deps(database, capfd): _check_json_output_deps(spec_list) +@pytest.mark.regression("52219") +def test_display_abstract_hash(): + spec = spack.spec.Spec("/foobar") + out = io.StringIO() + + spack.cmd.display_specs([spec], output=out) # errors on failure + assert "/foobar" in out.getvalue() + + @pytest.mark.db def test_find_format(database, config): output = find("--format", "{name}-{^mpi.name}", "mpileaks") @@ -275,15 +286,15 @@ def test_find_format_deps_paths(database, config): output == f"""\ mpileaks-2.3 {mpileaks.prefix} - callpath-1.0 {mpileaks['callpath'].prefix} - dyninst-8.2 {mpileaks['dyninst'].prefix} - libdwarf-20130729 {mpileaks['libdwarf'].prefix} - libelf-0.8.13 {mpileaks['libelf'].prefix} - compiler-wrapper-1.0 {mpileaks['compiler-wrapper'].prefix} - gcc-10.2.1 {mpileaks['gcc'].prefix} - gcc-runtime-10.2.1 {mpileaks['gcc-runtime'].prefix} - zmpi-1.0 {mpileaks['zmpi'].prefix} - fake-1.0 {mpileaks['fake'].prefix} + callpath-1.0 {mpileaks["callpath"].prefix} + dyninst-8.2 {mpileaks["dyninst"].prefix} + libdwarf-20130729 {mpileaks["libdwarf"].prefix} + libelf-0.8.13 {mpileaks["libelf"].prefix} + compiler-wrapper-1.0 {mpileaks["compiler-wrapper"].prefix} + gcc-10.2.1 {mpileaks["gcc"].prefix} + gcc-runtime-10.2.1 {mpileaks["gcc-runtime"].prefix} + zmpi-1.0 {mpileaks["zmpi"].prefix} + fake-1.0 {mpileaks["fake"].prefix} """ ) @@ -380,7 +391,7 @@ def test_find_specs_include_concrete_env( with ev.read("combined_env"): output = find() - assert "No root specs" in output + assert "no root specs" in output assert "Included specs" in output assert "mpileaks" in output assert "libelf" in output @@ -417,7 +428,7 @@ def test_find_specs_nested_include_concrete_env( with ev.read("test3"): output = find() - assert "No root specs" in output + assert "no root specs" in output assert "Included specs" in output assert "mpileaks" in output assert "libelf" in output @@ -536,3 +547,52 @@ def test_find_based_on_commit_sha(mock_git_version_info, monkeypatch): install("--fake", f"git-test-commit commit={commits[0]}") output = find(f"commit={commits[0]}") assert "git-test-commit" in output + + +@pytest.mark.usefixtures("mock_packages") +@pytest.mark.parametrize( + "spack_yaml,expected,not_expected", + [ + ( + """ +spack: + specs: + - mpileaks + - group: extras + specs: + - libelf +""", + [ + "2 root specs", + # Group names + "extras", + "default", + # root specs + "mpileaks", + "libelf", + ], + [], + ), + ( + """ +spack: + specs: + - group: tools + specs: + - libelf +""", + ["1 root spec", "tools", "libelf"], + ["1 root specs", "default"], + ), + ], +) +def test_find_env_with_groups(spack_yaml, expected, not_expected, tmp_path: pathlib.Path): + """Tests that the output of spack find contains expected matches when using an + environment with groups. + """ + (tmp_path / "spack.yaml").write_text(spack_yaml) + with ev.Environment(tmp_path): + output = find() + + assert all(x in output for x in expected) + assert all(x not in output for x in not_expected) diff --git a/lib/spack/spack/test/cmd/gc.py b/lib/spack/spack/test/cmd/gc.py index f061010b6dadd9..34b7a4a94174e1 100644 --- a/lib/spack/spack/test/cmd/gc.py +++ b/lib/spack/spack/test/cmd/gc.py @@ -165,3 +165,69 @@ def test_gc_except_specific_dir_env( assert "Restricting garbage collection" not in output assert "Successfully uninstalled zmpi" in output assert not mutable_database.query_local("zmpi") + + +@pytest.fixture +def mock_installed_environment(mutable_database, mutable_mock_env_path): + + def _create_environment(name, spack_yaml): + tmp_env = ev.create(name) + spack_yaml_path = pathlib.Path(tmp_env.path) / "spack.yaml" + spack_yaml_path.write_text(spack_yaml) + e = ev.read(name) + with ev.read(name): + e.concretize() + e.install_all(fake=True) + e.write() + return e + + return _create_environment + + +@pytest.mark.db +@pytest.mark.parametrize( + "explicit,expected_explicit,expected_implicit", + [ + (True, ["gcc@14.0.1", "openblas", "dyninst"], []), + (False, ["dyninst"], ["gcc@14.0.1", "openblas"]), + ], +) +def test_gc_with_explicit_groups( + explicit, expected_explicit, expected_implicit, mutable_database, mock_installed_environment +): + """Tests the semantics of the "explicit" attribute of environment groups""" + e = mock_installed_environment( + "test_gc_explicit", + f""" +spack: + config: + installer: new + specs: + - group: base + explicit: {explicit} + specs: + - gcc@14.0.1 + - openblas + - group: apps + needs: [base] + specs: + - dyninst %c=gcc@14.0.1 +""", + ) + + # Test DB status + for query in expected_explicit: + assert mutable_database.query_local(query, explicit=True) + + for query in expected_implicit: + assert mutable_database.query_local(query, explicit=False) + + with e: + output = gc("-y") + + # Test gc behavior + for query in expected_implicit: + assert f"Successfully uninstalled {query}" in output + + for query in expected_explicit: + assert f"Successfully uninstalled {query}" not in output diff --git a/lib/spack/spack/test/cmd/gpg.py b/lib/spack/spack/test/cmd/gpg.py index db95110699fd41..5a745e10f1b945 100644 --- a/lib/spack/spack/test/cmd/gpg.py +++ b/lib/spack/spack/test/cmd/gpg.py @@ -33,7 +33,7 @@ ], ) def test_find_gpg(cmd_name, version, tmp_path: pathlib.Path, mock_gnupghome, monkeypatch): - TEMPLATE = "#!/bin/sh\n" 'echo "{version}"\n' + TEMPLATE = '#!/bin/sh\necho "{version}"\n' with fs.working_dir(str(tmp_path)): for fname in (cmd_name, "gpgconf"): diff --git a/lib/spack/spack/test/cmd/info.py b/lib/spack/spack/test/cmd/info.py index 812a6630d17e70..099692460b953e 100644 --- a/lib/spack/spack/test/cmd/info.py +++ b/lib/spack/spack/test/cmd/info.py @@ -7,12 +7,19 @@ import pytest from spack.main import SpackCommand, SpackCommandError +from spack.repo import UnknownPackageError pytestmark = [pytest.mark.usefixtures("mock_packages")] info = SpackCommand("info") +def test_package_suggestion(): + with pytest.raises(UnknownPackageError) as exc_info: + info("vtk") + assert "Did you mean one of the following packages?" in str(exc_info.value) + + def test_deprecated_option_warns(): info("--variants-by-name", "vtk-m") assert "--variants-by-name is deprecated" in info.output @@ -43,7 +50,8 @@ def test_is_externally_detectable(pkg_query, expected): @pytest.mark.parametrize( - "pkg_query", ["vtk-m", "gcc"] # This should ensure --test's c_names processing loop covered + "pkg_query", + ["vtk-m", "gcc"], # This should ensure --test's c_names processing loop covered ) @pytest.mark.parametrize("extra_args", [[], ["--by-name"]]) def test_info_fields(pkg_query, extra_args): diff --git a/lib/spack/spack/test/cmd/install.py b/lib/spack/spack/test/cmd/install.py index c3a6307fb80950..ebcbbd36c38e59 100644 --- a/lib/spack/spack/test/cmd/install.py +++ b/lib/spack/spack/test/cmd/install.py @@ -50,7 +50,12 @@ def noop(*args, **kwargs): def test_install_package_and_dependency( - tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery + tmp_path: pathlib.Path, + mock_packages, + mock_archive, + mock_fetch, + install_mockery, + installer_variant, ): log = "test" with fs.working_dir(str(tmp_path)): @@ -81,24 +86,29 @@ def _check_runtests_all(pkg): @pytest.mark.disable_clean_stage_check def test_install_runtests_notests(monkeypatch, mock_packages, install_mockery): - monkeypatch.setattr(spack.package_base.PackageBase, "unit_test_check", _check_runtests_none) + monkeypatch.setattr(spack.package_base.PackageBase, "_unit_test_check", _check_runtests_none) install("-v", "dttop") @pytest.mark.disable_clean_stage_check def test_install_runtests_root(monkeypatch, mock_packages, install_mockery): - monkeypatch.setattr(spack.package_base.PackageBase, "unit_test_check", _check_runtests_dttop) + monkeypatch.setattr(spack.package_base.PackageBase, "_unit_test_check", _check_runtests_dttop) install("--test=root", "dttop") @pytest.mark.disable_clean_stage_check def test_install_runtests_all(monkeypatch, mock_packages, install_mockery): - monkeypatch.setattr(spack.package_base.PackageBase, "unit_test_check", _check_runtests_all) + monkeypatch.setattr(spack.package_base.PackageBase, "_unit_test_check", _check_runtests_all) install("--test=all", "pkg-a") def test_install_package_already_installed( - tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery + tmp_path: pathlib.Path, + mock_packages, + mock_archive, + mock_fetch, + install_mockery, + installer_variant, ): with fs.working_dir(str(tmp_path)): install("--fake", "libdwarf") @@ -184,7 +194,9 @@ def test_install_output_on_python_error(mock_packages, mock_archive, mock_fetch, @pytest.mark.disable_clean_stage_check -def test_install_with_source(mock_packages, mock_archive, mock_fetch, install_mockery): +def test_install_with_source( + mock_packages, mock_archive, mock_fetch, install_mockery, installer_variant +): """Verify that source has been copied into place.""" install("--source", "--keep-stage", "trivial-install-test-package") spec = spack.concretize.concretize_one("trivial-install-test-package") @@ -194,7 +206,9 @@ def test_install_with_source(mock_packages, mock_archive, mock_fetch, install_mo ) -def test_install_env_variables(mock_packages, mock_archive, mock_fetch, install_mockery): +def test_install_env_variables( + mock_packages, mock_archive, mock_fetch, install_mockery, installer_variant +): spec = spack.concretize.concretize_one("pkg-c") install("pkg-c") assert os.path.isfile(spec.package.install_env_path) @@ -213,7 +227,9 @@ def test_show_log_on_error(mock_packages, mock_archive, mock_fetch, install_mock assert "See build log for details:" in out -def test_install_overwrite(mock_packages, mock_archive, mock_fetch, install_mockery): +def test_install_overwrite( + mock_packages, mock_archive, mock_fetch, install_mockery, installer_variant +): """Tests installing a spec, and then re-installing it in the same prefix.""" spec = spack.concretize.concretize_one("pkg-c") install("pkg-c") @@ -245,7 +261,9 @@ def test_install_overwrite(mock_packages, mock_archive, mock_fetch, install_mock assert fs.hash_directory(spec.prefix, ignore=ignores) != bad_md5 -def test_install_overwrite_not_installed(mock_packages, mock_archive, mock_fetch, install_mockery): +def test_install_overwrite_not_installed( + mock_packages, mock_archive, mock_fetch, install_mockery, installer_variant +): """Tests that overwrite doesn't fail if the package is not installed""" spec = spack.concretize.concretize_one("pkg-c") assert not os.path.exists(spec.prefix) @@ -265,7 +283,7 @@ def test_install_commit(mock_git_version_info, install_mockery, mock_packages, m monkeypatch.setattr(spack.package_base.PackageBase, "git", file_url, raising=False) - # Use the earliest commit in the respository + # Use the earliest commit in the repository spec = spack.concretize.concretize_one(f"git-test-commit@{commits[-1]}") PackageInstaller([spec.package], explicit=True).install() @@ -277,7 +295,9 @@ def test_install_commit(mock_git_version_info, install_mockery, mock_packages, m assert content == "[0]" # contents are weird for another test -def test_install_overwrite_multiple(mock_packages, mock_archive, mock_fetch, install_mockery): +def test_install_overwrite_multiple( + mock_packages, mock_archive, mock_fetch, install_mockery, installer_variant +): # Try to install a spec and then to reinstall it. libdwarf = spack.concretize.concretize_one("libdwarf") cmake = spack.concretize.concretize_one("cmake") @@ -351,7 +371,7 @@ def test_install_invalid_spec(): "exc_typename,msg", [("RuntimeError", "something weird happened"), ("ValueError", "spec is not concrete")], ) -def test_junit_output_with_failures(tmp_path: pathlib.Path, exc_typename, msg): +def test_junit_output_with_failures(tmp_path: pathlib.Path, exc_typename, msg, installer_variant): with fs.working_dir(str(tmp_path)): install( "--verbose", @@ -363,9 +383,11 @@ def test_junit_output_with_failures(tmp_path: pathlib.Path, exc_typename, msg): fail_on_error=False, ) - assert isinstance(install.error, spack.build_environment.ChildError) - assert install.error.name == exc_typename - assert install.error.pkg.name == "raiser" + # New installer considers Python exceptions ordinary build failures. + if installer_variant == "old": + assert isinstance(install.error, spack.build_environment.ChildError) + assert install.error.name == exc_typename + assert install.error.pkg.name == "raiser" files = list(tmp_path.iterdir()) filename = tmp_path / "test.xml" @@ -421,7 +443,7 @@ def test_junit_output_with_errors( tmp_path: pathlib.Path, monkeypatch, ): - throw = _keyboard_error if expected_exc == KeyboardInterrupt else _runtime_error + throw = _keyboard_error if expected_exc is KeyboardInterrupt else _runtime_error monkeypatch.setattr(spack.installer.BuildTask, "complete", throw) with fs.working_dir(str(tmp_path)): @@ -485,7 +507,9 @@ def test_install_mix_cli_and_files(spec_format, clispecs, filespecs, tmp_path: p assert install.returncode == 0 -def test_extra_files_are_archived(mock_packages, mock_archive, mock_fetch, install_mockery): +def test_extra_files_are_archived( + mock_packages, mock_archive, mock_fetch, install_mockery, installer_variant +): s = spack.concretize.concretize_one("archive-files") install("archive-files") @@ -500,7 +524,7 @@ def test_extra_files_are_archived(mock_packages, mock_archive, mock_fetch, insta @pytest.mark.disable_clean_stage_check def test_cdash_report_concretization_error( - tmp_path: pathlib.Path, mock_fetch, install_mockery, conflict_spec + tmp_path: pathlib.Path, mock_fetch, install_mockery, conflict_spec, installer_variant ): with fs.working_dir(str(tmp_path)): with pytest.raises(SpackError): @@ -519,7 +543,9 @@ def test_cdash_report_concretization_error( @pytest.mark.not_on_windows("Windows log_output logs phase header out of order") @pytest.mark.disable_clean_stage_check -def test_cdash_upload_build_error(capfd, tmp_path: pathlib.Path, mock_fetch, install_mockery): +def test_cdash_upload_build_error( + capfd, tmp_path: pathlib.Path, mock_fetch, install_mockery, installer_variant +): with fs.working_dir(str(tmp_path)): with pytest.raises(SpackError): install( @@ -537,7 +563,9 @@ def test_cdash_upload_build_error(capfd, tmp_path: pathlib.Path, mock_fetch, ins @pytest.mark.disable_clean_stage_check -def test_cdash_upload_clean_build(tmp_path: pathlib.Path, mock_fetch, install_mockery): +def test_cdash_upload_clean_build( + tmp_path: pathlib.Path, mock_fetch, install_mockery, installer_variant +): with fs.working_dir(str(tmp_path)): install("--log-file=cdash_reports", "--log-format=cdash", "pkg-c") report_dir = tmp_path / "cdash_reports" @@ -550,7 +578,9 @@ def test_cdash_upload_clean_build(tmp_path: pathlib.Path, mock_fetch, install_mo @pytest.mark.disable_clean_stage_check -def test_cdash_upload_extra_params(tmp_path: pathlib.Path, mock_fetch, install_mockery): +def test_cdash_upload_extra_params( + tmp_path: pathlib.Path, mock_fetch, install_mockery, installer_variant +): with fs.working_dir(str(tmp_path)): install( "--log-file=cdash_reports", @@ -571,7 +601,9 @@ def test_cdash_upload_extra_params(tmp_path: pathlib.Path, mock_fetch, install_m @pytest.mark.disable_clean_stage_check -def test_cdash_buildstamp_param(tmp_path: pathlib.Path, mock_fetch, install_mockery): +def test_cdash_buildstamp_param( + tmp_path: pathlib.Path, mock_fetch, install_mockery, installer_variant +): with fs.working_dir(str(tmp_path)): cdash_track = "some_mocked_track" buildstamp_format = f"%Y%m%d-%H%M-{cdash_track}" @@ -592,7 +624,12 @@ def test_cdash_buildstamp_param(tmp_path: pathlib.Path, mock_fetch, install_mock @pytest.mark.disable_clean_stage_check def test_cdash_install_from_spec_json( - tmp_path: pathlib.Path, mock_fetch, install_mockery, mock_packages, mock_archive + tmp_path: pathlib.Path, + mock_fetch, + install_mockery, + mock_packages, + mock_archive, + installer_variant, ): with fs.working_dir(str(tmp_path)): spec_json_path = str(tmp_path / "spec.json") @@ -644,24 +681,30 @@ def test_build_warning_output(mock_fetch, install_mockery): assert "foo.c:89: warning: some weird warning!" in e.value.long_message -def test_cache_only_fails(mock_fetch, install_mockery): +@pytest.mark.disable_clean_stage_check # new installer keeps a log for build cache installs +def test_cache_only_fails(mock_fetch, install_mockery, installer_variant): # libelf from cache fails to install, which automatically removes the # the libdwarf build task out = install("--cache-only", "libdwarf", fail_on_error=False) - - assert "Failed to install gcc-runtime" in out - assert "Skipping build of libdwarf" in out - assert "was not installed" in out - - # Check that failure prefix locks are still cached - failed_packages = [ - pkg_name for dag_hash, pkg_name in spack.store.STORE.failure_tracker.locker.locks.keys() - ] - assert "libelf" in failed_packages - assert "libdwarf" in failed_packages + assert isinstance(install.error, spack.error.InstallError) + assert not spack.store.STORE.db.query_local("libdwarf") + assert not spack.store.STORE.db.query_local("libelf") + + if installer_variant == "old": + assert "Failed to install gcc-runtime" in out + assert "Skipping build of libdwarf" in out + assert "was not installed" in out + + # Check that failure prefix locks are still cached + failed_packages = [ + pkg_name + for dag_hash, pkg_name in spack.store.STORE.failure_tracker.locker.locks.keys() + ] + assert "libelf" in failed_packages + assert "libdwarf" in failed_packages -def test_install_only_dependencies(mock_fetch, install_mockery): +def test_install_only_dependencies(mock_fetch, install_mockery, installer_variant): dep = spack.concretize.concretize_one("dependency-install") root = spack.concretize.concretize_one("dependent-install") @@ -682,7 +725,7 @@ def test_install_only_package(mock_fetch, install_mockery): assert "1 uninstalled dependency" in msg -def test_install_deps_then_package(mock_fetch, install_mockery): +def test_install_deps_then_package(mock_fetch, install_mockery, installer_variant): dep = spack.concretize.concretize_one("dependency-install") root = spack.concretize.concretize_one("dependent-install") @@ -697,7 +740,9 @@ def test_install_deps_then_package(mock_fetch, install_mockery): # Unit tests should not be affected by the user's managed environments @pytest.mark.not_on_windows("Environment views not supported on windows. Revisit after #34701") @pytest.mark.regression("12002") -def test_install_only_dependencies_in_env(mutable_mock_env_path, mock_fetch, install_mockery): +def test_install_only_dependencies_in_env( + mutable_mock_env_path, mock_fetch, install_mockery, installer_variant +): env("create", "test") with ev.read("test"): @@ -713,7 +758,7 @@ def test_install_only_dependencies_in_env(mutable_mock_env_path, mock_fetch, ins # Unit tests should not be affected by the user's managed environments @pytest.mark.regression("12002") def test_install_only_dependencies_of_all_in_env( - mutable_mock_env_path, mock_fetch, install_mockery + mutable_mock_env_path, mock_fetch, install_mockery, installer_variant ): env("create", "--without-view", "test") @@ -735,7 +780,7 @@ def test_install_only_dependencies_of_all_in_env( # Unit tests should not be affected by the user's managed environments def test_install_no_add_in_env( - tmp_path: pathlib.Path, mutable_mock_env_path, mock_fetch, install_mockery + tmp_path: pathlib.Path, mutable_mock_env_path, mock_fetch, install_mockery, installer_variant ): # To test behavior of --add option, we create the following environment: # @@ -852,7 +897,9 @@ def test_install_help_cdash(): @pytest.mark.disable_clean_stage_check -def test_cdash_auth_token(tmp_path: pathlib.Path, mock_fetch, install_mockery, monkeypatch): +def test_cdash_auth_token( + tmp_path: pathlib.Path, mock_fetch, install_mockery, monkeypatch, installer_variant +): with fs.working_dir(str(tmp_path)): monkeypatch.setenv("SPACK_CDASH_AUTH_TOKEN", "asdf") out = install("--fake", "-v", "--log-file=cdash_reports", "--log-format=cdash", "pkg-a") @@ -861,7 +908,9 @@ def test_cdash_auth_token(tmp_path: pathlib.Path, mock_fetch, install_mockery, m @pytest.mark.not_on_windows("Windows log_output logs phase header out of order") @pytest.mark.disable_clean_stage_check -def test_cdash_configure_warning(tmp_path: pathlib.Path, mock_fetch, install_mockery): +def test_cdash_configure_warning( + tmp_path: pathlib.Path, mock_fetch, install_mockery, installer_variant +): with fs.working_dir(str(tmp_path)): # Test would fail if install raised an error. @@ -909,7 +958,7 @@ def test_install_fails_no_args_suggests_env_activation(tmp_path: pathlib.Path): # Unit tests should not be affected by the user's managed environments @pytest.mark.not_on_windows("Environment views not supported on windows. Revisit after #34701") def test_install_env_with_tests_all( - mutable_mock_env_path, mock_packages, mock_fetch, install_mockery + mutable_mock_env_path, mock_packages, mock_fetch, install_mockery, installer_variant ): env("create", "test") with ev.read("test"): @@ -922,7 +971,7 @@ def test_install_env_with_tests_all( # Unit tests should not be affected by the user's managed environments @pytest.mark.not_on_windows("Environment views not supported on windows. Revisit after #34701") def test_install_env_with_tests_root( - mutable_mock_env_path, mock_packages, mock_fetch, install_mockery + mutable_mock_env_path, mock_packages, mock_fetch, install_mockery, installer_variant ): env("create", "test") with ev.read("test"): @@ -934,7 +983,9 @@ def test_install_env_with_tests_root( # Unit tests should not be affected by the user's managed environments @pytest.mark.not_on_windows("Environment views not supported on windows. Revisit after #34701") -def test_install_empty_env(mutable_mock_env_path, mock_packages, mock_fetch, install_mockery): +def test_install_empty_env( + mutable_mock_env_path, mock_packages, mock_fetch, install_mockery, installer_variant +): env_name = "empty" env("create", env_name) with ev.read(env_name): diff --git a/lib/spack/spack/test/cmd/list.py b/lib/spack/spack/test/cmd/list.py index ee45f163a91e5d..8f1d8de5ac5eb3 100644 --- a/lib/spack/spack/test/cmd/list.py +++ b/lib/spack/spack/test/cmd/list.py @@ -7,9 +7,12 @@ import pytest +import spack.cmd.list import spack.paths import spack.repo +import spack.util.git from spack.main import SpackCommand +from spack.test.conftest import RepoBuilder pytestmark = [pytest.mark.usefixtures("mock_packages")] @@ -63,6 +66,8 @@ def test_list_format_version_json(): output = list("--format", "version_json") assert '{"name": "zmpi",' in output assert '{"name": "dyninst",' in output + assert "packages/zmpi/package.py" in output + import json json.loads(output) @@ -75,6 +80,83 @@ def test_list_format_html(): assert '
    ' in output assert "

    hdf5" in output + assert "packages/hdf5/package.py" in output + + +@pytest.mark.parametrize( + "url", + [ + "git@github.com:username/spack-packages.git", + "https://github.com/username/spack-packages.git", + "git@github.com:username/spack.git", + "https://github.com/username/spack.git", + ], +) +def test_list_url_schemes(mock_util_executable, url): + """Confirm the command handles supported repository URLs.""" + pkg_name = "hdf5" + + _, _, registered_responses = mock_util_executable + registered_responses["config"] = url + registered_responses["rev-parse"] = f"path/to/builtin/packages/{pkg_name}/" + + output = list("--format", "version_json", pkg_name) + assert f"{registered_responses['rev-parse']}package.py" in output + assert os.path.basename(url).replace(".git", "") in output + + +def test_list_format_local_repo(tmp_path: pathlib.Path): + """Confirm a file path is returned for local repository.""" + pkg_name = "mypkg" + repo_root = tmp_path / "repos" / "spack_repo" / "builtin" + repo_root.mkdir(parents=True) + (repo_root / "repo.yaml").write_text("repo:\n namespace: builtin\n api: v2.2\n") + package_root = repo_root / "packages" / pkg_name + package_root.mkdir(parents=True) + (package_root / "package.py").write_text( + """\ +from spack.package import * + +class Mypkg(Package): + pass +""" + ) + + test_repo = spack.repo.from_path(str(repo_root)) + with spack.repo.use_repositories(test_repo): + # Confirm a path is returned when fail to retrieve the remote origin URL + output = list("--format", "version_json", pkg_name) + assert "github.com" not in output + assert f"packages/{pkg_name}/package.py" in output + + +def test_list_format_non_github_repo(tmp_path: pathlib.Path, mock_util_executable): + """Confirm a file path is returned for a non-github repository.""" + pkg_name = "mypkg" + repo_root = tmp_path / "my" / "project" / "spack_repo" / "builtin" + repo_root.mkdir(parents=True) + (repo_root / "repo.yaml").write_text("repo:\n namespace: builtin\n api: v2.2\n") + package_root = repo_root / "packages" / pkg_name + package_root.mkdir(parents=True) + package_path = package_root / "package.py" + package_path.write_text( + """\ +from spack.package import * + +class Mypkg(Package): + pass +""" + ) + + test_repo = spack.repo.from_path(str(repo_root)) + with spack.repo.use_repositories(test_repo): + # Confirm a path is returned for a non-standard spack repository + _, _, registered_responses = mock_util_executable + registered_responses["config"] = "https://gitlab.com/username/my-packages.git" + registered_responses["rev-parse"] = str(package_root) + os.sep + + output = list("--format", "version_json", pkg_name) + assert package_path.as_uri() in output def test_list_update(tmp_path: pathlib.Path): @@ -140,3 +222,34 @@ def test_list_repos(): assert total_pkgs > mock_pkgs > builder_pkgs assert both_repos == total_pkgs + + +@pytest.mark.usefixtures("config") +def test_list_github_url_fails(repo_builder: RepoBuilder, monkeypatch): + with spack.repo.use_repositories(repo_builder.root): + repo_builder.add_package("pkg-a") + repo = spack.repo.PATH.repos[0] + pkg = repo.get_pkg_class("pkg-a") + + old_path = repo.python_path + try: + # Check that a repository with no python path has no URL + monkeypatch.setattr(repo, "python_path", None) + assert spack.cmd.list.github_url(pkg) is None, ( + "Expected no python path means unable to determine the repo URL" + ) + + # Check that a repository path that doesn't exist has no URL + monkeypatch.setattr(repo, "python_path", "/repo/root/does/not/exists") + assert spack.cmd.list.github_url(pkg) is None, ( + "Expected bad repo path means unable to determine the repo URL" + ) + finally: + monkeypatch.setattr(repo, "python_path", old_path) + + # Check that missing git results in the file path + monkeypatch.setattr(spack.util.git, "git", lambda: None) + filepath = spack.cmd.list.github_url(pkg) + assert filepath and filepath.startswith("file://"), ( + "Expected missing 'git' results in a file URI" + ) diff --git a/lib/spack/spack/test/cmd/location.py b/lib/spack/spack/test/cmd/location.py index 4a9f5830a39c16..cef2e7ac37b4d2 100644 --- a/lib/spack/spack/test/cmd/location.py +++ b/lib/spack/spack/test/cmd/location.py @@ -71,14 +71,19 @@ def test_location_source_dir_missing(): @pytest.mark.parametrize( - "options", - [([]), (["--source-dir", "mpileaks"]), (["--env", "missing-env"]), (["spec1", "spec2"])], + "options,expected_code", + [ + ([], 2), + (["--source-dir", "mpileaks"], 1), + (["--env", "missing-env"], 1), + (["spec1", "spec2"], 2), + ], ) -def test_location_cmd_error(options): +def test_location_cmd_error(options, expected_code): """Ensure the proper error is raised with problematic location options.""" with pytest.raises(spack.main.SpackCommandError) as e: location(*options) - assert e.value.code == 1 + assert e.value.code == expected_code def test_location_env_exists(mutable_mock_env_path): @@ -104,6 +109,80 @@ def test_location_env_missing(): assert out == error +def test_location_active_view(mutable_mock_env_path, monkeypatch): + """Tests spack location --view for the active view.""" + mutable_mock_env_path.mkdir() + view_path = os.path.abspath(mutable_mock_env_path / "path" / "to" / "view") + spack_yaml = mutable_mock_env_path / ev.manifest_name + spack_yaml.write_text( + f"""spack: + specs: [] + view: + viewname: + root: {view_path} + concretizer: + unify: True + """ + ) + e = ev.Environment(mutable_mock_env_path) + monkeypatch.setenv(ev.spack_env_view_var, "viewname") + with e: + assert location("--view").strip() == view_path + + +def test_location_no_active_view(mutable_mock_env_path): + """Tests spack location --env without active view.""" + mutable_mock_env_path.mkdir() + view_path = os.path.abspath(mutable_mock_env_path / "path" / "to" / "view") + spack_yaml = mutable_mock_env_path / ev.manifest_name + spack_yaml.write_text( + f"""spack: + specs: [] + view: + viewname: + root: {view_path} + concretizer: + unify: True + """ + ) + e = ev.Environment(mutable_mock_env_path) + error = "==> Error: no active view in the current environment" + with e: + out = location("--view", fail_on_error=False).strip() + assert out == error + + +def test_location_view_exists(mutable_mock_env_path): + """Tests spack location --view for an existing view.""" + mutable_mock_env_path.mkdir() + view_path = os.path.abspath(mutable_mock_env_path / "path" / "to" / "view") + spack_yaml = mutable_mock_env_path / ev.manifest_name + spack_yaml.write_text( + f"""spack: + specs: [] + view: + viewname: + root: {view_path} + concretizer: + unify: True + """ + ) + e = ev.Environment(mutable_mock_env_path) + with e: + assert location("--view", "viewname").strip() == view_path + + +def test_location_view_missing(mutable_mock_env_path): + """Tests spack location --env with missing view.""" + e = ev.create("example", with_view=True) + e.write() + missing_view_name = "missing-view" + error = "==> Error: no such view in the current environment: '%s'" % missing_view_name + with e: + out = location("--view", missing_view_name, fail_on_error=False).strip() + assert out == error + + @pytest.mark.db @pytest.mark.not_on_windows("Broken on Windows") def test_location_install_dir(mock_spec): @@ -135,12 +214,13 @@ def test_location_paths_options(option, expected): @pytest.mark.parametrize( "specs,expected", - [([], "You must supply a spec."), (["spec1", "spec2"], "Too many specs. Supply only one.")], + [([], "requires a spec"), (["spec1", "spec2"], "too many specs, supply only one")], ) def test_location_spec_errors(specs, expected): """Tests spack location with bad spec options.""" - error = "==> Error: %s" % expected - assert location(*specs, fail_on_error=False).strip() == error + output = location(*specs, fail_on_error=False) + assert expected in output + assert location.returncode == 2 @pytest.mark.db diff --git a/lib/spack/spack/test/cmd/logs.py b/lib/spack/spack/test/cmd/logs.py index 1004306c24e715..8b0b7ddb752d56 100644 --- a/lib/spack/spack/test/cmd/logs.py +++ b/lib/spack/spack/test/cmd/logs.py @@ -14,6 +14,7 @@ import spack.cmd.logs import spack.concretize import spack.error +import spack.main import spack.spec from spack.main import SpackCommand @@ -53,8 +54,9 @@ def test_logs_cmd_errors(install_mockery, mock_fetch, mock_archive, mock_package with pytest.raises(spack.error.SpackError, match="is not installed or staged"): logs("pkg-c") - with pytest.raises(spack.error.SpackError, match="Too many specs"): + with pytest.raises(spack.main.SpackCommandError) as e: logs("pkg-c mpi") + assert e.value.code == 2 install("pkg-c") os.remove(spec.package.install_log_path) diff --git a/lib/spack/spack/test/cmd/maintainers.py b/lib/spack/spack/test/cmd/maintainers.py index 6c040016280ce5..28a93cbb412cb8 100644 --- a/lib/spack/spack/test/cmd/maintainers.py +++ b/lib/spack/spack/test/cmd/maintainers.py @@ -15,6 +15,7 @@ MAINTAINED_PACKAGES = [ "gcc-runtime", + "invalid-maintainer", "maintainers-1", "maintainers-2", "maintainers-3", @@ -43,6 +44,9 @@ def test_all(): assert out == [ "gcc-runtime:", "haampie", + "invalid-maintainer:", + "github_user1,", + "github_user2", "maintainers-1:", "user1,", "user2", @@ -66,6 +70,10 @@ def test_all(): def test_all_by_user(): out = split(maintainers("--all", "--by-user")) assert out == [ + "github_user1:", + "invalid-maintainer", + "github_user2:", + "invalid-maintainer", "haampie:", "gcc-runtime", "user0:", diff --git a/lib/spack/spack/test/cmd/mirror.py b/lib/spack/spack/test/cmd/mirror.py index a6dd5605cb34d6..083b3d23b0ef6f 100644 --- a/lib/spack/spack/test/cmd/mirror.py +++ b/lib/spack/spack/test/cmd/mirror.py @@ -12,7 +12,7 @@ import spack.concretize import spack.config import spack.environment as ev -import spack.error +import spack.mirrors.utils import spack.package_base import spack.spec import spack.util.git @@ -302,7 +302,7 @@ def test_mirror_remove_by_scope(mutable_config, tmp_path: pathlib.Path): assert "mock" in system_output # Confirm that when the scope is not specified, it is removed from top scope - mirror("add", "--scope=site", "mock", str(tmp_path / "mockrepo")) + mirror("add", "--scope=site", "mock", str(tmp_path / "mock_mirror")) mirror("remove", "mock") site_output = mirror("list", "--scope=site") system_output = mirror("list", "--scope=system") @@ -520,27 +520,19 @@ def test_all_specs_with_all_versions_dont_concretize(self): @pytest.mark.parametrize( "cli_args,error_str", [ - # Passed more than one among -f --all - ( - {"specs": None, "file": "input.txt", "all": True}, - "cannot specify specs with a file if", - ), - ( - {"specs": "hdf5", "file": "input.txt", "all": False}, - "cannot specify specs with a file AND", - ), - ({"specs": None, "file": None, "all": False}, "no packages were specified"), - # Passed -n along with --all + (["create", "--file", "input.txt", "--all"], "cannot specify specs with a file if"), + (["create", "--file", "input.txt", "hdf5"], "cannot specify specs with a file AND"), + (["create"], "no packages were specified"), ( - {"specs": None, "file": None, "all": True, "versions_per_spec": 2}, + ["create", "--all", "--versions-per-spec", "2"], "cannot specify '--versions_per-spec'", ), ], ) def test_error_conditions(self, cli_args, error_str): - args = MockMirrorArgs(**cli_args) - with pytest.raises(spack.error.SpackError, match=error_str): - spack.cmd.mirror.mirror_create(args) + output = mirror(*cli_args, fail_on_error=False) + assert error_str in output + assert mirror.returncode == 2 @pytest.mark.parametrize( "cli_args,not_expected", @@ -753,3 +745,20 @@ def test_git_provenance_relative_to_mirror( spec_head = spack.concretize.concretize_one(f"git-test-commit@main commit={head_commit}") assert spec_head.variants["commit"].value == head_commit + + +@pytest.mark.usefixtures("mock_packages") +def test_mirror_skip_placeholder_pkg(tmp_path: pathlib.Path): + """Test a placeholder package which should skip during mirror all""" + from spack.repo import PATH + + spec = spack.spec.Spec("placeholder@1.5") + pkg_cls = PATH.get_pkg_class(spec.name) + pkg_obj = pkg_cls(spec) + mirror_cache = spack.mirrors.utils.get_mirror_cache(str(tmp_path)) + mirror_stats = spack.mirrors.utils.MirrorStatsForOneSpec(spec) + result = spack.mirrors.utils.create_mirror_from_package_object( + pkg_obj, mirror_cache, mirror_stats + ) + assert result is False + assert not mirror_stats.errors diff --git a/lib/spack/spack/test/cmd/print_shell_vars.py b/lib/spack/spack/test/cmd/print_shell_vars.py index 866d4a3da9e46c..d410a4633aa51a 100644 --- a/lib/spack/spack/test/cmd/print_shell_vars.py +++ b/lib/spack/spack/test/cmd/print_shell_vars.py @@ -23,23 +23,3 @@ def test_print_shell_vars_csh(capfd): assert "set _sp_tcl_roots = " in out assert "set _sp_lmod_roots = " in out assert "set _sp_module_prefix = " not in out - - -def test_print_shell_vars_sh_modules(capfd): - print_setup_info("sh", "modules") - out, _ = capfd.readouterr() - - assert "_sp_sys_type=" in out - assert "_sp_tcl_roots=" in out - assert "_sp_lmod_roots=" in out - assert "_sp_module_prefix=" in out - - -def test_print_shell_vars_csh_modules(capfd): - print_setup_info("csh", "modules") - out, _ = capfd.readouterr() - - assert "set _sp_sys_type = " in out - assert "set _sp_tcl_roots = " in out - assert "set _sp_lmod_roots = " in out - assert "set _sp_module_prefix = " in out diff --git a/lib/spack/spack/test/cmd/python.py b/lib/spack/spack/test/cmd/python.py index 883fe8c42ad58c..9d56d9a9c3c88a 100644 --- a/lib/spack/spack/test/cmd/python.py +++ b/lib/spack/spack/test/cmd/python.py @@ -39,4 +39,5 @@ def test_python_with_module(): def test_python_raises(): out = python("--foobar", fail_on_error=False) - assert "Error: Unknown arguments" in out + assert python.returncode == 2 + assert "--foobar" in out diff --git a/lib/spack/spack/test/cmd/repo.py b/lib/spack/spack/test/cmd/repo.py index 1aa874dca66ec6..a91ac6f115d7a9 100644 --- a/lib/spack/spack/test/cmd/repo.py +++ b/lib/spack/spack/test/cmd/repo.py @@ -84,7 +84,7 @@ def test_repo_remove_by_scope(mutable_config, tmp_path: pathlib.Path): def test_env_repo_path_vars_substitution( tmp_path: pathlib.Path, install_mockery, mutable_mock_env_path, monkeypatch ): - """Test Spack correctly substitues repo paths with environment variables when creating an + """Test Spack correctly substitutes repo paths with environment variables when creating an environment from a manifest file.""" monkeypatch.setenv("CUSTOM_REPO_PATH", ".") @@ -813,6 +813,82 @@ def test_repo_list_format_flags( assert config_names_lines == ["monorepo", "uninitialized", "misconfigured"] +def test_repo_list_json_output(mutable_config: spack.config.Configuration, tmp_path: pathlib.Path): + """Test the --json flag for repo list command. + + This test verifies that: + 1. The --json flag produces valid JSON output + 2. The output contains the expected repository information + 3. Different repository types (installed, uninitialized, error) + are correctly represented + """ + import json + + # Fake a git monorepo with two package repositories + monorepo_path = tmp_path / "monorepo" + (monorepo_path / ".git").mkdir(parents=True) + repo("create", str(monorepo_path), "repo_one") + repo("create", str(monorepo_path), "repo_two") + + # Configure repositories in Spack + test_repos = { + # git repo that provides two package repositories + "monorepo": { + "git": "https://example.com/monorepo.git", + "destination": str(monorepo_path), + "paths": ["spack_repo/repo_one", "spack_repo/repo_two"], + }, + # git repo that is not yet cloned + "uninitialized": { + "git": "https://example.com/uninitialized.git", + "destination": str(tmp_path / "uninitialized"), + }, + # invalid local repository + "misconfigured": str(tmp_path / "misconfigured"), + } + mutable_config.set("repos", test_repos, scope="site") + + # Get and parse JSON output + json_output = repo("list", "--json") + repo_data = json.loads(json_output) + + # Verify we got a list of repositories + assert isinstance(repo_data, list), "Expected JSON output to be a list" + + # Index repositories by namespace for easier validation + repos_by_namespace = {} + for item in repo_data: + # Check all required fields are present + required_fields = ["name", "namespace", "path", "api_version", "status", "error"] + for field in required_fields: + assert field in item, f"Repository missing required field: {field}" + + # Store by namespace for later validation + repos_by_namespace[item["namespace"]] = item + + # Verify installed repositories (repo_one and repo_two) + for namespace in ["repo_one", "repo_two"]: + assert namespace in repos_by_namespace, f"Missing repository: {namespace}" + repo_info = repos_by_namespace[namespace] + assert repo_info["name"] == "monorepo", f"Incorrect name for {namespace}" + assert repo_info["status"] == "installed", f"Incorrect status for {namespace}" + assert repo_info["error"] is None, f"Unexpected error for {namespace}" + assert repo_info["api_version"], f"Missing API version for {namespace}" + + # Verify uninitialized repository + assert "uninitialized" in repos_by_namespace, "Missing uninitialized repository" + uninit_repo = repos_by_namespace["uninitialized"] + assert uninit_repo["name"] == "uninitialized", "Incorrect name for uninitialized repo" + assert uninit_repo["status"] == "uninitialized", "Incorrect status for uninitialized repo" + + # Verify misconfigured repository + assert "misconfigured" in repos_by_namespace, "Missing misconfigured repository" + misc_repo = repos_by_namespace["misconfigured"] + assert misc_repo["name"] == "misconfigured", "Incorrect name for misconfigured repo" + assert misc_repo["status"] == "error", "Incorrect status for misconfigured repo" + assert misc_repo["error"] is not None, "Missing error message for misconfigured repo" + + @pytest.mark.parametrize( "repo_name,flags", [ @@ -864,3 +940,112 @@ def test_repo_update_invalid_flags(monkeypatch, mutable_config, flags): with pytest.raises(SpackError): repo("update", *flags) + + +def test_repo_show_version_updates_no_changes(mock_git_package_changes): + """Test that show-version-updates handles empty results gracefully""" + test_repo, _, commits = mock_git_package_changes + + with spack.repo.use_repositories(test_repo): + # Use the same commit for both refs - no changes + output = repo("show-version-updates", test_repo.root, commits[-1], commits[-1]) + + # Should have warning message + assert "No packages were added or changed" in output + + # Should not have any specs + assert "diff-test@" not in output + + +def test_repo_show_version_updates_success(mock_git_package_changes): + """Test that show-version-updates successfully outputs the correct specs""" + test_repo, _, commits = mock_git_package_changes + + with spack.repo.use_repositories(test_repo): + # commits are ordered from newest to oldest after reversal + # commits[-2] = add v2.1.5, commits[-4] = add v2.1.7 and v2.1.8 + # Find versions added between these commits + # Includes v2.1.6 (git version), v2.1.7, and v2.1.8 (sha256 versions) + output = repo("show-version-updates", test_repo.root, commits[-2], commits[-4]) + + # Verify all three versions are included + assert "diff-test@" in output + assert "2.1.6" in output + assert "2.1.7" in output + assert "2.1.8" in output + + # Should have three specs + lines = [ + line.strip() + for line in output.strip().split("\n") + if line.strip() and "Warning" not in line + ] + assert len(lines) == 3 + + +def test_repo_show_version_updates_excludes_manual_packages(monkeypatch, mock_git_package_changes): + """Test --no-manual-packages flag excludes packages with manual_download=True""" + test_repo, _, commits = mock_git_package_changes + + with spack.repo.use_repositories(test_repo): + # Set manual_download=True on the package + pkg_class = spack.repo.PATH.get_pkg_class("diff-test") + monkeypatch.setattr(pkg_class, "manual_download", True) + + # Run show-version-updates with --no-manual-packages flag + output = repo( + "show-version-updates", + "--no-manual-packages", + test_repo.root, + commits[-2], + commits[-4], + ) + + # Package should be excluded + assert "diff-test@" not in output + assert "No packages were added or changed" in output + + +def test_repo_show_version_updates_excludes_non_redistributable( + monkeypatch, mock_git_package_changes +): + """Test --only-redistributable flag excludes packages if redistribute_source returns False""" + test_repo, _, commits = mock_git_package_changes + + with spack.repo.use_repositories(test_repo): + # Set redistribute_source to return False + pkg_class = spack.repo.PATH.get_pkg_class("diff-test") + monkeypatch.setattr(pkg_class, "redistribute_source", classmethod(lambda cls, spec: False)) + + # Run show-version-updates with --only-redistributable flag + output = repo( + "show-version-updates", + "--only-redistributable", + test_repo.root, + commits[-2], + commits[-4], + ) + + # Package should be excluded + assert "diff-test@" not in output + assert "No new package versions found" in output + + +def test_repo_show_version_updates_excludes_git_versions(mock_git_package_changes): + """Test --no-git-versions flag excludes versions from git (tag/commit)""" + test_repo, _, commits = mock_git_package_changes + + with spack.repo.use_repositories(test_repo): + # commits[-3] = add v2.1.6 (git version), commits[-4] = add v2.1.7 and v2.1.8 (sha256) + # Without --no-git-versions, v2.1.6 would be included + output = repo( + "show-version-updates", "--no-git-versions", test_repo.root, commits[-3], commits[-4] + ) + + # v2.1.6 (git version) should be excluded + assert "2.1.6" not in output + + # v2.1.7 and v2.1.8 (sha256 versions) should be included + assert "diff-test@" in output + assert "2.1.7" in output + assert "2.1.8" in output diff --git a/lib/spack/spack/test/cmd/spec.py b/lib/spack/spack/test/cmd/spec.py index a8cb2d88de9ef3..ed04784f2ef9d3 100644 --- a/lib/spack/spack/test/cmd/spec.py +++ b/lib/spack/spack/test/cmd/spec.py @@ -32,7 +32,7 @@ def test_spec(): assert "mpich@3.0.4" in output -def test_spec_concretizer_args(mutable_database, do_not_check_runtimes_on_reuse): +def test_spec_concretizer_args(mutable_database): """End-to-end test of CLI concretizer prefs. It's here to make sure that everything works from CLI @@ -140,7 +140,7 @@ def test_spec_deptypes_edges(): def test_spec_returncode(): with pytest.raises(SpackCommandError): spec() - assert spec.returncode == 1 + assert spec.returncode == 2 def test_spec_parse_error(): @@ -171,7 +171,7 @@ def test_env_aware_spec(mutable_mock_env_path): [ ("develop-branch-version", "f3c7206350ac8ee364af687deaae5c574dcfca2c=develop", None), ("develop-branch-version", "git." + "a" * 40 + "=develop", None), - ("callpath", "f3c7206350ac8ee364af687deaae5c574dcfca2c=1.0", spack.error.FetchError), + ("callpath", "f3c7206350ac8ee364af687deaae5c574dcfca2c=1.0", spack.error.PackageError), ("develop-branch-version", "git.foo=0.2.15", None), ], ) diff --git a/lib/spack/spack/test/cmd/stage.py b/lib/spack/spack/test/cmd/stage.py index cfaf2fa1edc287..25858d2b04166f 100644 --- a/lib/spack/spack/test/cmd/stage.py +++ b/lib/spack/spack/test/cmd/stage.py @@ -179,6 +179,6 @@ def is_installed(self): specs_to_stage = [s for s in all_specs if not filter(s)] specs_were_filtered = [skip not in specs_to_stage for skip in should_be_filtered] - assert all( - specs_were_filtered - ), f"Packages associated with bools: {[s.name for s in should_be_filtered]}" + assert all(specs_were_filtered), ( + f"Packages associated with bools: {[s.name for s in should_be_filtered]}" + ) diff --git a/lib/spack/spack/test/cmd/style.py b/lib/spack/spack/test/cmd/style.py index fb983bf3fc657e..8b78e9287773d8 100644 --- a/lib/spack/spack/test/cmd/style.py +++ b/lib/spack/spack/test/cmd/style.py @@ -25,10 +25,11 @@ style = spack.main.SpackCommand("style") +pytestmark = pytest.mark.skipif( + sys.platform == "win32", reason="CI uses cross drive paths that raise errors with relpath" +) -ISORT = which("isort") -BLACK = which("black") -FLAKE8 = which("flake8") +RUFF = which("ruff") MYPY = which("mypy") @@ -41,15 +42,15 @@ def has_develop_branch(git): @pytest.fixture(scope="function") -def flake8_package(tmp_path: pathlib.Path): +def ruff_package(tmp_path: pathlib.Path): """Style only checks files that have been modified. This fixture makes a small - change to the ``flake8`` mock package, yields the filename, then undoes the + change to the ``ruff`` mock package, yields the filename, then undoes the change on cleanup. """ repo = spack.repo.from_path(spack.paths.mock_packages_path) - filename = repo.filename_for_package_name("flake8") + filename = repo.filename_for_package_name("ruff") rel_path = os.path.dirname(os.path.relpath(filename, spack.paths.prefix)) - tmp = tmp_path / rel_path / "flake8-ci-package.py" + tmp = tmp_path / rel_path / "ruff-ci-package.py" tmp.parent.mkdir(parents=True, exist_ok=True) tmp.touch() tmp = str(tmp) @@ -61,19 +62,19 @@ def flake8_package(tmp_path: pathlib.Path): @pytest.fixture -def flake8_package_with_errors(scope="function"): - """A flake8 package with errors.""" +def ruff_package_with_errors(scope="function"): + """A ruff package with errors.""" repo = spack.repo.from_path(spack.paths.mock_packages_path) - filename = repo.filename_for_package_name("flake8") + filename = repo.filename_for_package_name("ruff") tmp = filename + ".tmp" shutil.copy(filename, tmp) package = FileFilter(tmp) - # this is a black error (quote style and spacing before/after operator) + # this is a ruff error (quote style and spacing before/after operator) package.filter('state = "unmodified"', "state = 'modified'", string=True) - # this is an isort error (orderign) and a flake8 error (unused import) + # this is two ruff errors (unused import) (orderign) package.filter( "from spack.package import *", "from spack.package import *\nimport os", string=True ) @@ -91,13 +92,13 @@ def test_changed_files_from_git_rev_base(git, tmp_path: pathlib.Path): (tmp_path / "bin").mkdir(parents=True, exist_ok=True) (tmp_path / "bin" / "spack").touch() - assert changed_files(base="HEAD") == ["bin/spack"] - assert changed_files(base="main") == ["bin/spack"] + assert changed_files(base="HEAD") == [pathlib.Path("bin/spack")] + assert changed_files(base="main") == [pathlib.Path("bin/spack")] git("add", "bin/spack") git("commit", "--no-gpg-sign", "-m", "v1") assert changed_files(base="HEAD") == [] - assert changed_files(base="HEAD~") == ["bin/spack"] + assert changed_files(base="HEAD~") == [pathlib.Path("bin/spack")] def test_changed_no_base(git, tmp_path: pathlib.Path, capfd): @@ -140,7 +141,7 @@ def test_changed_files_all_files(mock_packages): # a mock package repo = spack.repo.from_path(spack.paths.mock_packages_path) - filename = repo.filename_for_package_name("flake8") + filename = repo.filename_for_package_name("ruff") assert filename in files # this test @@ -154,24 +155,12 @@ def test_bad_root(tmp_path: pathlib.Path): """Ensure that `spack style` doesn't run on non-spack directories.""" output = style("--root", str(tmp_path), fail_on_error=False) assert "This does not look like a valid spack root" in output - assert style.returncode != 0 - - -def test_style_is_package(): - """Ensure the is_package() function works.""" - assert spack.cmd.style.is_package( - "var/spack/repos/spack_repo/builtin/packages/hdf5/package.py" - ) - assert spack.cmd.style.is_package( - "var/spack/repos/spack_repo/builtin/packages/zlib/package.py" - ) - assert not spack.cmd.style.is_package("lib/spack/spack/spec.py") - assert not spack.cmd.style.is_package("lib/spack/external/pytest.py") + assert style.returncode == 1 @pytest.fixture -def external_style_root(git, flake8_package_with_errors, tmp_path: pathlib.Path): - """Create a mock git repository for running spack style.""" +def external_style_root(git, ruff_package_with_errors, tmp_path: pathlib.Path): + """Create a mock repository for running spack style.""" # create a sort-of spack-looking directory script = tmp_path / "bin" / "spack" script.parent.mkdir(parents=True, exist_ok=True) @@ -196,18 +185,12 @@ def external_style_root(git, flake8_package_with_errors, tmp_path: pathlib.Path) # copy the buggy package in py_file = spack_dir / "dummy.py" py_file.touch() - shutil.copy(flake8_package_with_errors, str(py_file)) - - # add the buggy file on the feature branch - with working_dir(str(tmp_path)): - git("add", str(py_file)) - git("commit", "--no-gpg-sign", "-m", "add new file") + shutil.copy(ruff_package_with_errors, str(py_file)) yield tmp_path, py_file -@pytest.mark.skipif(not ISORT, reason="isort is not installed.") -@pytest.mark.skipif(not BLACK, reason="black is not installed.") +@pytest.mark.skipif(not RUFF, reason="ruff is not installed.") def test_fix_style(external_style_root): """Make sure spack style --fix works.""" tmp_path, py_file = external_style_root @@ -219,16 +202,18 @@ def test_fix_style(external_style_root): shutil.copy(broken_dummy, broken_py) assert not filecmp.cmp(broken_py, fixed_py) - # black and isort are the tools that actually fix things - style("--root", str(tmp_path), "--tool", "isort,black", "--fix") - + # dummy.py is in the same directory and will raise errors unrelated to this + # check, don't fail on those errors, just check to make sure + # we fixed the intended file correctly + # Note: can't just specify the correct file due to cross drive issues on Windows + style( + "--root", str(tmp_path), "--tool", "ruff-check,ruff-format", "--fix", fail_on_error=False + ) assert filecmp.cmp(broken_py, fixed_py) -@pytest.mark.skipif(not FLAKE8, reason="flake8 is not installed.") -@pytest.mark.skipif(not ISORT, reason="isort is not installed.") +@pytest.mark.skipif(not RUFF, reason="ruff is not installed.") @pytest.mark.skipif(not MYPY, reason="mypy is not installed.") -@pytest.mark.skipif(not BLACK, reason="black is not installed.") def test_external_root(external_style_root): """Ensure we can run in a separate root directory w/o configuration files.""" tmp_path, py_file = external_style_root @@ -238,70 +223,69 @@ def test_external_root(external_style_root): output = style("--root-relative", "--root", str(tmp_path), fail_on_error=False) # make sure it failed - assert style.returncode != 0 + assert style.returncode == 1 - # isort error - assert "%s Imports are incorrectly sorted" % str(py_file) in output + # ruff-check error + assert "Import block is un-sorted or un-formatted\n --> lib/spack/spack/dummy.py" in output # mypy error - assert 'lib/spack/spack/dummy.py:47: error: Name "version" is not defined' in output + assert 'lib/spack/spack/dummy.py:45: error: Name "version" is not defined' in output - # black error + # ruff-format error assert "--- lib/spack/spack/dummy.py" in output assert "+++ lib/spack/spack/dummy.py" in output - # flake8 error - assert "lib/spack/spack/dummy.py:8: [F401] 'os' imported but unused" in output + # ruff-check error + assert "`os` imported but unused\n --> lib/spack/spack/dummy.py" in output -@pytest.mark.skipif(not FLAKE8, reason="flake8 is not installed.") -def test_style(flake8_package, tmp_path: pathlib.Path): - root_relative = os.path.relpath(flake8_package, spack.paths.prefix) +@pytest.mark.skipif(not RUFF, reason="ruff is not installed.") +def test_style(ruff_package, tmp_path: pathlib.Path): + root_relative = os.path.relpath(ruff_package, spack.paths.prefix) # use a working directory to test cwd-relative paths, as tests run in # the spack prefix by default with working_dir(str(tmp_path)): - relative = os.path.relpath(flake8_package) + relative = os.path.relpath(ruff_package) # one specific arg - output = style("--tool", "flake8", flake8_package, fail_on_error=False) + output = style("--tool", "ruff-check", ruff_package, fail_on_error=False) assert relative in output assert "spack style checks were clean" in output # specific file that isn't changed - output = style("--tool", "flake8", __file__, fail_on_error=False) + output = style("--tool", "ruff-check", __file__, fail_on_error=False) assert relative not in output assert __file__ in output assert "spack style checks were clean" in output # root-relative paths - output = style("--tool", "flake8", "--root-relative", flake8_package) + output = style("--tool", "ruff-check", "--root-relative", ruff_package) assert root_relative in output assert "spack style checks were clean" in output -@pytest.mark.skipif(not FLAKE8, reason="flake8 is not installed.") -def test_style_with_errors(flake8_package_with_errors): - root_relative = os.path.relpath(flake8_package_with_errors, spack.paths.prefix) +@pytest.mark.skipif(not RUFF, reason="ruff is not installed.") +def test_style_with_errors(ruff_package_with_errors): + root_relative = os.path.relpath(ruff_package_with_errors, spack.paths.prefix) output = style( - "--tool", "flake8", "--root-relative", flake8_package_with_errors, fail_on_error=False + "--tool", "ruff-check", "--root-relative", ruff_package_with_errors, fail_on_error=False ) assert root_relative in output - assert style.returncode != 0 + assert style.returncode == 1 assert "spack style found errors" in output -@pytest.mark.skipif(not BLACK, reason="black is not installed.") -@pytest.mark.skipif(not FLAKE8, reason="flake8 is not installed.") -def test_style_with_black(flake8_package_with_errors): - output = style("--tool", "black,flake8", flake8_package_with_errors, fail_on_error=False) - assert "black found errors" in output - assert style.returncode != 0 +@pytest.mark.skipif(not RUFF, reason="ruff is not installed.") +def test_style_with_ruff_format(ruff_package_with_errors): + output = style("--tool", "ruff-format", ruff_package_with_errors, fail_on_error=False) + assert "ruff-format found errors" in output + assert style.returncode == 1 assert "spack style found errors" in output def test_skip_tools(): - output = style("--skip", "import,isort,mypy,black,flake8") + output = style("--skip", "import,ruff-check,ruff-format,mypy") assert "Nothing to run" in output @@ -337,12 +321,12 @@ def something(y: spack.util.url.Url): ... root = str(tmp_path) output_buf = io.StringIO() exit_code = _run_import_check( - [str(file)], + [file], fix=False, out=output_buf, root_relative=False, - root=spack.paths.prefix, - working_dir=root, + root=pathlib.Path(spack.paths.prefix), + working_dir=pathlib.Path(root), ) output = output_buf.getvalue() @@ -360,12 +344,12 @@ def something(y: spack.util.url.Url): ... # run it with --fix, should have the same output. output_buf = io.StringIO() exit_code = _run_import_check( - [str(file)], + [file], fix=True, out=output_buf, root_relative=False, - root=spack.paths.prefix, - working_dir=root, + root=pathlib.Path(spack.paths.prefix), + working_dir=pathlib.Path(root), ) output = output_buf.getvalue() assert exit_code == 1 @@ -378,12 +362,12 @@ def something(y: spack.util.url.Url): ... # after fix a second fix is idempotent output_buf = io.StringIO() exit_code = _run_import_check( - [str(file)], + [file], fix=True, out=output_buf, root_relative=False, - root=spack.paths.prefix, - working_dir=root, + root=pathlib.Path(spack.paths.prefix), + working_dir=pathlib.Path(root), ) output = output_buf.getvalue() assert exit_code == 0 @@ -403,12 +387,12 @@ def test_run_import_check_syntax_error_and_missing(tmp_path: pathlib.Path): (tmp_path / "syntax-error.py").write_text("""this 'is n(ot python code""") output_buf = io.StringIO() exit_code = _run_import_check( - [str(tmp_path / "syntax-error.py"), str(tmp_path / "missing.py")], + [tmp_path / "syntax-error.py", tmp_path / "missing.py"], fix=False, out=output_buf, root_relative=True, - root=str(tmp_path), - working_dir=str(tmp_path / "does-not-matter"), + root=tmp_path, + working_dir=tmp_path / "does-not-matter", ) output = output_buf.getvalue() assert "syntax-error.py: could not parse" in output @@ -421,114 +405,12 @@ def test_case_sensitive_imports(tmp_path: pathlib.Path): (tmp_path / "lib" / "spack" / "example").mkdir(parents=True) (tmp_path / "lib" / "spack" / "example" / "__init__.py").write_text("class Example:\n pass") (tmp_path / "lib" / "spack" / "example" / "example.py").write_text("foo = 1") - assert spack.cmd.style._module_part(str(tmp_path), "example.Example") == "example" + assert spack.cmd.style._module_part(tmp_path, "example.Example") == "example" def test_pkg_imports(): - assert spack.cmd.style._module_part(spack.paths.prefix, "spack.pkg.builtin.boost") is None - assert spack.cmd.style._module_part(spack.paths.prefix, "spack.pkg") is None - - -def test_spec_strings(tmp_path: pathlib.Path): - (tmp_path / "example.py").write_text( - """\ -def func(x): - print("dont fix %s me" % x, 3) - return x.satisfies("+foo %gcc +bar") and x.satisfies("%gcc +baz") -""" - ) - (tmp_path / "example.json").write_text( - """\ -{ - "spec": [ - "+foo %gcc +bar~nope ^dep %clang +yup @3.2 target=x86_64 /abcdef ^another %gcc ", - "%gcc +baz" - ], - "%gcc x=y": 2 -} -""" - ) - (tmp_path / "example.yaml").write_text( - """\ -spec: - - "+foo %gcc +bar" - - "%gcc +baz" - - "this is fine %clang" -"%gcc x=y": 2 -""" - ) - - issues = set() - - def collect_issues(path: str, line: int, col: int, old: str, new: str): - issues.add((path, line, col, old, new)) - - # check for issues with custom handler - spack.cmd.style._check_spec_strings( - [ - str(tmp_path / "nonexistent.py"), - str(tmp_path / "example.py"), - str(tmp_path / "example.json"), - str(tmp_path / "example.yaml"), - ], - handler=collect_issues, - ) - - assert issues == { - ( - str(tmp_path / "example.json"), - 3, - 9, - "+foo %gcc +bar~nope ^dep %clang +yup @3.2 target=x86_64 /abcdef ^another %gcc ", - "+foo +bar~nope %gcc ^dep +yup @3.2 target=x86_64 /abcdef %clang ^another %gcc ", - ), - (str(tmp_path / "example.json"), 4, 9, "%gcc +baz", "+baz %gcc"), - (str(tmp_path / "example.json"), 6, 5, "%gcc x=y", "x=y %gcc"), - (str(tmp_path / "example.py"), 3, 23, "+foo %gcc +bar", "+foo +bar %gcc"), - (str(tmp_path / "example.py"), 3, 57, "%gcc +baz", "+baz %gcc"), - (str(tmp_path / "example.yaml"), 2, 5, "+foo %gcc +bar", "+foo +bar %gcc"), - (str(tmp_path / "example.yaml"), 3, 5, "%gcc +baz", "+baz %gcc"), - (str(tmp_path / "example.yaml"), 5, 1, "%gcc x=y", "x=y %gcc"), - } - - # fix the issues in the files - spack.cmd.style._check_spec_strings( - [ - str(tmp_path / "nonexistent.py"), - str(tmp_path / "example.py"), - str(tmp_path / "example.json"), - str(tmp_path / "example.yaml"), - ], - handler=spack.cmd.style._spec_str_fix_handler, - ) - - assert ( - (tmp_path / "example.json").read_text() - == """\ -{ - "spec": [ - "+foo +bar~nope %gcc ^dep +yup @3.2 target=x86_64 /abcdef %clang ^another %gcc ", - "+baz %gcc" - ], - "x=y %gcc": 2 -} -""" - ) - assert ( - (tmp_path / "example.py").read_text() - == """\ -def func(x): - print("dont fix %s me" % x, 3) - return x.satisfies("+foo +bar %gcc") and x.satisfies("+baz %gcc") -""" - ) assert ( - (tmp_path / "example.yaml").read_text() - == """\ -spec: - - "+foo +bar %gcc" - - "+baz %gcc" - - "this is fine %clang" -"x=y %gcc": 2 -""" + spack.cmd.style._module_part(pathlib.Path(spack.paths.prefix), "spack.pkg.builtin.boost") + is None ) + assert spack.cmd.style._module_part(pathlib.Path(spack.paths.prefix), "spack.pkg") is None diff --git a/lib/spack/spack/test/cmd/tags.py b/lib/spack/spack/test/cmd/tags.py index 150f9112d4239f..f005034c4d51f3 100644 --- a/lib/spack/spack/test/cmd/tags.py +++ b/lib/spack/spack/test/cmd/tags.py @@ -6,12 +6,11 @@ import spack.main import spack.repo from spack.installer import PackageInstaller -from spack.tag import TagIndex tags = spack.main.SpackCommand("tags") -def test_tags_bad_options(): +def test_tags_bad_options(mock_packages): out = tags("-a", "tag1", fail_on_error=False) assert "option OR provide" in out @@ -38,9 +37,10 @@ def test_tags_all_mock_tag_packages(mock_packages): assert pkg in out -def test_tags_no_tags(monkeypatch): - monkeypatch.setattr(spack.repo.PATH, "tag_index", TagIndex()) - out = tags() +def test_tags_no_tags(repo_builder): + repo_builder.add_package("pkg-a") + with spack.repo.use_repositories(repo_builder.root): + out = tags() assert "No tagged" in out diff --git a/lib/spack/spack/test/cmd/test.py b/lib/spack/spack/test/cmd/test.py index 609d1f79e718cc..2389d337d828a7 100644 --- a/lib/spack/spack/test/cmd/test.py +++ b/lib/spack/spack/test/cmd/test.py @@ -198,7 +198,6 @@ def test_test_list_all(mock_packages): assert set(pkgs) == { "py-numpy", "fail-test-audit", - "fail-test-audit-deprecated", "fail-test-audit-docstring", "fail-test-audit-impl", "mpich", diff --git a/lib/spack/spack/test/cmd/undevelop.py b/lib/spack/spack/test/cmd/undevelop.py index 711f1e8f708972..93a4197dfd88d6 100644 --- a/lib/spack/spack/test/cmd/undevelop.py +++ b/lib/spack/spack/test/cmd/undevelop.py @@ -43,6 +43,38 @@ def test_undevelop(tmp_path: pathlib.Path, mutable_config, mock_packages, mutabl assert not after.satisfies("dev_path=*") +def test_undevelop_all( + tmp_path: pathlib.Path, mutable_config, mock_packages, mutable_mock_env_path +): + # setup environment + envdir = tmp_path / "env" + envdir.mkdir() + with working_dir(str(envdir)): + with open("spack.yaml", "w", encoding="utf-8") as f: + f.write( + """\ +spack: + specs: + - mpich + + develop: + mpich: + spec: mpich@1.0 + path: /fake/path +""" + ) + + env("create", "test", "./spack.yaml") + with ev.read("test"): + before = spack.concretize.concretize_one("mpich") + undevelop("--all") + after = spack.concretize.concretize_one("mpich") + + # Removing dev spec from environment changes concretization + assert before.satisfies("dev_path=*") + assert not after.satisfies("dev_path=*") + + def test_undevelop_nonexistent( tmp_path: pathlib.Path, mutable_config, mock_packages, mutable_mock_env_path ): diff --git a/lib/spack/spack/test/cmd/verify.py b/lib/spack/spack/test/cmd/verify.py index 9ba39e1ff48593..f7e5b7cf6a15fc 100644 --- a/lib/spack/spack/test/cmd/verify.py +++ b/lib/spack/spack/test/cmd/verify.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for the `spack verify` command""" + import os import pathlib import platform diff --git a/lib/spack/spack/test/cmd_extensions.py b/lib/spack/spack/test/cmd_extensions.py index 93dcf527535d90..63c2e82ccbefd3 100644 --- a/lib/spack/spack/test/cmd_extensions.py +++ b/lib/spack/spack/test/cmd_extensions.py @@ -94,7 +94,7 @@ def hello_world(parser, args): @pytest.fixture(scope="function") def hello_world_cmd(hello_world_extension): - """Create and return an invokable "hello-world" extension command.""" + """Create and return an invocable "hello-world" extension command.""" yield spack.main.SpackCommand("hello-world") @@ -143,9 +143,7 @@ def hello(parser, args): hello_folks() elif args.subcommand == 'global': print(global_message) -""".format( - ext_pname=extension.pname - ), +""".format(ext_pname=extension.pname), ) init_file = extension.main / "__init__.py" diff --git a/lib/spack/spack/test/compilers/conversion.py b/lib/spack/spack/test/compilers/conversion.py index 2d77319773badf..deb3b5fd1a3fe4 100644 --- a/lib/spack/spack/test/compilers/conversion.py +++ b/lib/spack/spack/test/compilers/conversion.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests conversions from compilers.yaml""" + import pathlib import pytest diff --git a/lib/spack/spack/test/compilers/libraries.py b/lib/spack/spack/test/compilers/libraries.py index 74591af4230c1d..6b042de4c9cc7e 100644 --- a/lib/spack/spack/test/compilers/libraries.py +++ b/lib/spack/spack/test/compilers/libraries.py @@ -94,6 +94,7 @@ def module(*args): return "" elif args[0] == "load": monkeypatch.setenv("MODULE_LOADED", "1") + monkeypatch.setenv("LOADEDMODULES", "turn_on") monkeypatch.setattr(spack.util.module_cmd, "module", module) @@ -125,3 +126,20 @@ def test_compiler_environment(self, working_env, mock_gcc, monkeypatch): detector = spack.compilers.libraries.CompilerPropertyDetector(mock_gcc) with detector.compiler_environment(): assert os.environ["TEST"] == "yes" + + @pytest.mark.not_on_windows("Module files are not supported on Windows") + def test_compiler_invalid_module_raises(self, working_env, mock_gcc, monkeypatch): + """Test if an exception is raised when a module cannot be loaded""" + + def mock_load_module(module_name): + # Simulate module load failure + raise spack.util.module_cmd.ModuleLoadError(module_name) + + monkeypatch.setattr(spack.util.module_cmd, "load_module", mock_load_module) + + mock_gcc.external_modules = ["non_existent"] + detector = spack.compilers.libraries.CompilerPropertyDetector(mock_gcc) + + with pytest.raises(spack.util.module_cmd.ModuleLoadError): + with detector.compiler_environment(): + pass diff --git a/lib/spack/spack/test/concretization/compiler_runtimes.py b/lib/spack/spack/test/concretization/compiler_runtimes.py index 8343494ea18b14..35f46795ffa344 100644 --- a/lib/spack/spack/test/concretization/compiler_runtimes.py +++ b/lib/spack/spack/test/concretization/compiler_runtimes.py @@ -16,7 +16,7 @@ import spack.solver.asp import spack.spec from spack.environment.environment import ViewDescriptor -from spack.solver.reuse import SpecFilter, create_external_parser +from spack.solver.reuse import create_external_parser, spec_filter_from_packages_yaml from spack.solver.runtimes import external_config_with_implicit_externals from spack.version import Version @@ -25,7 +25,7 @@ def _concretize_with_reuse(*, root_str, reused_str, config): reused_spec = spack.concretize.concretize_one(reused_str) packages_with_externals = external_config_with_implicit_externals(config) completion_mode = config.get("concretizer:externals:completion") - external_specs = SpecFilter.from_packages_yaml( + external_specs = spec_filter_from_packages_yaml( external_parser=create_external_parser(packages_with_externals, completion_mode), packages_with_externals=packages_with_externals, include=[], diff --git a/lib/spack/spack/test/concretization/core.py b/lib/spack/spack/test/concretization/core.py index 548a774ddd23e8..41f4241b83d2e3 100644 --- a/lib/spack/spack/test/concretization/core.py +++ b/lib/spack/spack/test/concretization/core.py @@ -34,9 +34,11 @@ import spack.repo import spack.solver.asp import spack.solver.core +import spack.solver.input_analysis import spack.solver.reuse import spack.solver.runtimes import spack.spec +import spack.spec_filter import spack.util.file_cache import spack.util.hash import spack.util.spack_yaml as syaml @@ -44,7 +46,7 @@ from spack.externals import ExternalDependencyError from spack.installer import PackageInstaller from spack.solver.asp import Result -from spack.solver.reuse import SpecFilter, create_external_parser +from spack.solver.reuse import create_external_parser, spec_filter_from_packages_yaml from spack.solver.runtimes import external_config_with_implicit_externals from spack.spec import Spec from spack.test.conftest import RepoBuilder @@ -327,7 +329,7 @@ def weights_from_result(result: Result, *, name: str) -> Dict[str, int]: # This must use the mutable_config fixture because the test # adjusting_default_target_based_on_compiler uses the current_host fixture, # which changes the config. -@pytest.mark.usefixtures("mutable_config", "mock_packages", "do_not_check_runtimes_on_reuse") +@pytest.mark.usefixtures("mutable_config", "mock_packages") class TestConcretize: def test_concretize(self, spec): check_concretize(spec) @@ -497,6 +499,10 @@ def test_disable_mixing_prevents_mixing(self): with pytest.raises(spack.error.UnsatisfiableSpecError): spack.concretize.concretize_one("dt-diamond%clang ^dt-diamond-bottom%gcc") + def test_disable_mixing_is_per_language(self): + with spack.config.override("concretizer", {"compiler_mixing": False}): + spack.concretize.concretize_one("openblas %c=llvm %fortran=gcc") + def test_disable_mixing_override_by_package(self): with spack.config.override("concretizer", {"compiler_mixing": ["dt-diamond-bottom"]}): root = spack.concretize.concretize_one("dt-diamond%clang ^dt-diamond-bottom%gcc") @@ -564,6 +570,29 @@ def test_disable_mixing_allow_compiler_link(self): assert x.satisfies("%c=gcc") assert "llvm" in x + def test_compiler_run_dep_link_dep_not_forced(self, temporary_store): + """When a compiler is used as a pure build dependency, its transitive run-reachable deps + are unified in the build environment, but their pure link-type dependencies must NOT be + forced onto the package. + + Scenario: compiler-with-deps has a run+link dep on binutils-for-test, which has a pure + link dep on zlib. A package that depends on zlib and uses compiler-with-deps as its C + compiler should be free to pick its own zlib (here: zlib@1.2.8) independently of the + toolchain's zlib (zlib@1.2.11). Without the fix the imposed hash from binutils-for-test + forces the toolchain version onto the package, causing a conflict. + """ + # Pre-install the compiler with its transitive deps binutils-for-test and zlib@1.2.11 + compiler = spack.concretize.concretize_one("compiler-with-deps ^zlib@1.2.11") + assert compiler["zlib"].satisfies("@1.2.11") + PackageInstaller([compiler.package], fake=True, explicit=True).install() + + # Concretize a package that depends on a different zlib from its compiler's toolchain. + pkg = spack.concretize.concretize_one( + "pkg-with-zlib-dep %c=compiler-with-deps ^zlib@1.2.8" + ) + + assert pkg["zlib"].satisfies("@1.2.8") + def test_disable_mixing_env( self, mutable_mock_env_path, tmp_path: pathlib.Path, mock_packages, mutable_config ): @@ -654,11 +683,15 @@ def test_concretize_two_virtuals_with_dual_provider(self): """ spack.concretize.concretize_one("hypre ^openblas-with-lapack") - def test_concretize_two_virtuals_with_dual_provider_and_a_conflict(self): + @pytest.mark.parametrize("max_dupes_default", [1, 2, 3]) + def test_concretize_two_virtuals_with_dual_provider_and_a_conflict( + self, max_dupes_default, mutable_config + ): """Test a package with multiple virtual dependencies and force a provider that provides both, and another conflicting package that provides one. """ + mutable_config.set("concretizer:duplicates:max_dupes:default", max_dupes_default) s = Spec("hypre ^openblas-with-lapack ^netlib-lapack") with pytest.raises(spack.error.SpackError): spack.concretize.concretize_one(s) @@ -754,7 +787,7 @@ def test_concretize_propagate_one_variant(self): def test_concretize_propagate_through_first_level_deps(self): """Test that boolean valued variants can be propagated past first level - dependecies even if the first level dependency does have the variant""" + dependencies even if the first level dependency does have the variant""" spec = Spec("parent-foo-bar-fee ++fee") spec = spack.concretize.concretize_one(spec) @@ -792,7 +825,7 @@ def test_concretize_propagate_single_valued_variant(self): def test_concretize_propagate_multivalue_variant(self): """Test that multivalue variants are propagating the specified value(s) - to their dependecies. The dependencies should not have the default value""" + to their dependencies. The dependencies should not have the default value""" spec = Spec("multivalue-variant foo==baz,fee") spec = spack.concretize.concretize_one(spec) @@ -1911,7 +1944,7 @@ def test_best_effort_coconcretize(self, specs, checks): assert len(matches) == expected_count @pytest.mark.parametrize( - "specs,expected_spec,occurances", + "specs,expected_spec,occurrences", [ # The algorithm is greedy, and it might decide to solve the "best" # spec early in which case reuse is suboptimal. In this case the most @@ -1940,7 +1973,7 @@ def test_best_effort_coconcretize(self, specs, checks): (["hdf5+mpi", "zmpi", "mpich"], "mpich", 2), ], ) - def test_best_effort_coconcretize_preferences(self, specs, expected_spec, occurances): + def test_best_effort_coconcretize_preferences(self, specs, expected_spec, occurrences): """Test package preferences during coconcretization.""" specs = [Spec(s) for s in specs] solver = spack.solver.asp.Solver() @@ -1953,7 +1986,7 @@ def test_best_effort_coconcretize_preferences(self, specs, expected_spec, occura for spec in concrete_specs.values(): if expected_spec in spec: counter += 1 - assert counter == occurances, concrete_specs + assert counter == occurrences, concrete_specs def test_solve_in_rounds_all_unsolved(self, monkeypatch, mock_packages): specs = [Spec(x) for x in ["libdwarf%gcc", "libdwarf%clang"]] @@ -2006,7 +2039,7 @@ def test_version_weight_and_provenance(self, mutable_config): packages_with_externals = external_config_with_implicit_externals(mutable_config) completion_mode = mutable_config.get("concretizer:externals:completion") - external_specs = SpecFilter.from_packages_yaml( + external_specs = spec_filter_from_packages_yaml( external_parser=create_external_parser(packages_with_externals, completion_mode), packages_with_externals=packages_with_externals, include=[], @@ -2040,7 +2073,7 @@ def test_variant_penalty(self, mutable_config): """Test package preferences during concretization.""" packages_with_externals = external_config_with_implicit_externals(mutable_config) completion_mode = mutable_config.get("concretizer:externals:completion") - external_specs = SpecFilter.from_packages_yaml( + external_specs = spec_filter_from_packages_yaml( external_parser=create_external_parser(packages_with_externals, completion_mode), packages_with_externals=packages_with_externals, include=[], @@ -2396,7 +2429,7 @@ def test_result_specs_is_not_empty(self, mutable_config, specs): specs = [Spec(s) for s in specs] packages_with_externals = external_config_with_implicit_externals(mutable_config) completion_mode = mutable_config.get("concretizer:externals:completion") - external_specs = SpecFilter.from_packages_yaml( + external_specs = spec_filter_from_packages_yaml( external_parser=create_external_parser(packages_with_externals, completion_mode), packages_with_externals=packages_with_externals, include=[], @@ -2962,6 +2995,26 @@ def test_no_multiple_solutions_with_different_edges_same_nodes(self): assert len(edges) == 1 assert edges[0].spec.satisfies("@=60") + def test_build_environment_is_unified(self): + """A pure build dep that is marked build-tool can creates its own unification set. This + test ensures that its sibling build dependencies are unified with it, together with their + runtime dependencies. It ensures the same package cannot appear multiple times in a single + build environment, for example when it's both a direct build dep, as well as pulled in as + a transitive runtime dep of a sibling build dep.""" + spack.config.CONFIG.set("concretizer:duplicates", {"max_dupes": {"unify-build-deps-c": 2}}) + + # Fails because unify-build-deps-c version @1 and @2 are needed in the build environment + with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): + spack.concretize.concretize_one("unify-build-deps-a@1.0") + + # Succeeds because unify-build-deps-c version @2 is not needed in the build environment + spack.concretize.concretize_one("unify-build-deps-a@2.0") + + # Lastly, a sanity check that max_dupes is a requirement for this to work. + spack.config.CONFIG.set("concretizer:duplicates", {"max_dupes": {"unify-build-deps-c": 1}}) + with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): + spack.concretize.concretize_one("unify-build-deps-a@2.0") + @pytest.mark.regression("43647") def test_specifying_different_versions_build_deps(self): """Tests that we can concretize a spec with nodes using the same build @@ -3240,7 +3293,7 @@ def test_concretization_version_order(): ), ], ) -@pytest.mark.usefixtures("mutable_database", "mock_store", "do_not_check_runtimes_on_reuse") +@pytest.mark.usefixtures("mutable_database", "mock_store") @pytest.mark.not_on_windows("Expected length is different on Windows") def test_filtering_reused_specs( roots, reuse_yaml, expected, not_expected, expected_length, mutable_config @@ -3253,7 +3306,7 @@ def test_filtering_reused_specs( ) completion_mode = mutable_config.get("concretizer:externals:completion") selector = spack.solver.asp.ReusableSpecsSelector( - mutable_config, + configuration=mutable_config, external_parser=create_external_parser(packages_with_externals, completion_mode), packages_with_externals=packages_with_externals, ) @@ -3285,9 +3338,7 @@ def test_filtering_reused_specs( ], ) @pytest.mark.not_on_windows("Expected length is different on Windows") -def test_selecting_reused_sources( - reuse_yaml, expected_length, mutable_config, do_not_check_runtimes_on_reuse -): +def test_selecting_reused_sources(reuse_yaml, expected_length, mutable_config): """Tests that we can turn on/off sources of reusable specs""" # Assume all specs have a runtime dependency mutable_config.set("concretizer:reuse", reuse_yaml) @@ -3296,7 +3347,7 @@ def test_selecting_reused_sources( ) completion_mode = mutable_config.get("concretizer:externals:completion") selector = spack.solver.asp.ReusableSpecsSelector( - mutable_config, + configuration=mutable_config, external_parser=create_external_parser(packages_with_externals, completion_mode), packages_with_externals=packages_with_externals, ) @@ -3319,14 +3370,14 @@ def test_selecting_reused_sources( def test_spec_filters(specs, include, exclude, expected): specs = [Spec(x) for x in specs] expected = [Spec(x) for x in expected] - f = spack.solver.reuse.SpecFilter( + f = spack.spec_filter.SpecFilter( factory=lambda: specs, is_usable=lambda x: True, include=include, exclude=exclude ) assert f.selected_specs() == expected @pytest.mark.regression("38484") -def test_git_ref_version_can_be_reused(install_mockery, do_not_check_runtimes_on_reuse): +def test_git_ref_version_can_be_reused(install_mockery): first_spec = spack.concretize.concretize_one( spack.spec.Spec("git-ref-package@git.2.1.5=2.1.5~opt") ) @@ -3347,9 +3398,7 @@ def test_git_ref_version_can_be_reused(install_mockery, do_not_check_runtimes_on @pytest.mark.parametrize("standard_version", ["2.0.0", "2.1.5", "2.1.6"]) -def test_reuse_prefers_standard_over_git_versions( - standard_version, install_mockery, do_not_check_runtimes_on_reuse -): +def test_reuse_prefers_standard_over_git_versions(standard_version, install_mockery): """ order matters in this test. typically reuse would pick the highest versioned installed match but we want to prefer the standard version over git ref based versions @@ -3395,7 +3444,7 @@ def test_parallel_concretization(mutable_config, mock_packages): assert {s.name for s, _ in result} == {"pkg-a", "pkg-b"} -@pytest.mark.usefixtures("mutable_config", "mock_packages", "do_not_check_runtimes_on_reuse") +@pytest.mark.usefixtures("mutable_config", "mock_packages") @pytest.mark.parametrize( "spec_str, error_type", [ @@ -3415,7 +3464,7 @@ def test_spec_containing_commit_variant(spec_str, error_type): spack.concretize.concretize_one(spec) -@pytest.mark.usefixtures("mutable_config", "mock_packages", "do_not_check_runtimes_on_reuse") +@pytest.mark.usefixtures("mutable_config", "mock_packages") @pytest.mark.parametrize( "spec_str", [ @@ -3435,7 +3484,7 @@ def test_spec_with_commit_interacts_with_lookup(mock_git_version_info, monkeypat spack.concretize.concretize_one(spec) -@pytest.mark.usefixtures("mutable_config", "mock_packages", "do_not_check_runtimes_on_reuse") +@pytest.mark.usefixtures("mutable_config", "mock_packages") @pytest.mark.parametrize("version_str", [f"git.{'a' * 40}=main", "git.2.1.5=main"]) def test_relationship_git_versions_and_commit_variant(version_str): """ @@ -3450,7 +3499,7 @@ def test_relationship_git_versions_and_commit_variant(version_str): assert "commit" not in spec.variants -@pytest.mark.usefixtures("install_mockery", "do_not_check_runtimes_on_reuse") +@pytest.mark.usefixtures("install_mockery") def test_abstract_commit_spec_reuse(): commit = "abcd" * 10 spec_str_1 = f"git-ref-package@develop commit={commit}" @@ -3463,7 +3512,7 @@ def test_abstract_commit_spec_reuse(): assert spec2.dag_hash() == spec1.dag_hash() -@pytest.mark.usefixtures("install_mockery", "do_not_check_runtimes_on_reuse") +@pytest.mark.usefixtures("install_mockery") @pytest.mark.parametrize( "installed_commit, incoming_commit, reusable", [("a" * 40, "b" * 40, False), (None, "b" * 40, False), ("a" * 40, None, True)], @@ -3606,9 +3655,9 @@ def test_compiler_match_for_externals_is_taken_into_account( libelf: externals: - spec: "libelf@0.8.12 %gcc@10" - prefix: {tmp_path / 'gcc'} + prefix: {tmp_path / "gcc"} - spec: "libelf@0.8.13 %clang" - prefix: {tmp_path / 'clang'} + prefix: {tmp_path / "clang"} """ ) mutable_config.set("packages", packages_yaml["packages"]) @@ -3640,9 +3689,9 @@ def test_compiler_match_for_externals_with_versions( buildable: false externals: - spec: "libelf@0.8.12 %gcc@10" - prefix: {tmp_path / 'libelf-gcc10'} + prefix: {tmp_path / "libelf-gcc10"} - spec: "libelf@0.8.13 %gcc@9.4.0" - prefix: {tmp_path / 'libelf-gcc9'} + prefix: {tmp_path / "libelf-gcc9"} """ ) mutable_config.set("packages", packages_yaml["packages"]) @@ -3679,7 +3728,7 @@ def test_specifying_compilers_with_virtuals_syntax(default_mock_concretization): @pytest.mark.regression("49847") @pytest.mark.xfail(sys.platform == "win32", reason="issues with install mockery") -def test_reuse_when_input_specifies_build_dep(install_mockery, do_not_check_runtimes_on_reuse): +def test_reuse_when_input_specifies_build_dep(install_mockery): """Test that we can reuse a spec when specifying build dependencies in the input""" pkgb_old = spack.concretize.concretize_one(spack.spec.Spec("pkg-b@0.9 %gcc@9")) PackageInstaller([pkgb_old.package], fake=True, explicit=True).install() @@ -3697,9 +3746,7 @@ def test_reuse_when_input_specifies_build_dep(install_mockery, do_not_check_runt @pytest.mark.regression("49847") -def test_reuse_when_requiring_build_dep( - install_mockery, do_not_check_runtimes_on_reuse, mutable_config -): +def test_reuse_when_requiring_build_dep(install_mockery, mutable_config): """Test that we can reuse a spec when specifying build dependencies in requirements""" mutable_config.set("packages:all:require", "%gcc") pkgb_old = spack.concretize.concretize_one(spack.spec.Spec("pkg-b@0.9")) @@ -3748,7 +3795,7 @@ def test_installing_external_with_compilers_directly( buildable: false externals: - spec: {spec_str} - prefix: {tmp_path / 'libelf'} + prefix: {tmp_path / "libelf"} """ ) mutable_config.set("packages", packages_yaml["packages"]) @@ -3771,7 +3818,7 @@ def test_using_externals_with_compilers(mutable_config, mock_packages, tmp_path: buildable: false externals: - spec: libelf@0.8.12 %gcc@10 - prefix: {tmp_path / 'libelf'} + prefix: {tmp_path / "libelf"} """ ) mutable_config.set("packages", packages_yaml["packages"]) @@ -3786,9 +3833,7 @@ def test_using_externals_with_compilers(mutable_config, mock_packages, tmp_path: @pytest.mark.regression("50161") -def test_installed_compiler_and_better_external( - install_mockery, do_not_check_runtimes_on_reuse, mutable_config -): +def test_installed_compiler_and_better_external(install_mockery, mutable_config): """Tests that we always prefer a higher-priority external compiler, when we have a lower-priority compiler installed, and we try to concretize a spec without specifying the compiler dependency. @@ -3820,17 +3865,17 @@ def test_concrete_multi_valued_variants_in_externals( buildable: false externals: - spec: gcc@12.1.0 languages:='c,c++' - prefix: {tmp_path / 'gcc-12'} + prefix: {tmp_path / "gcc-12"} extra_attributes: compilers: - c: {tmp_path / 'gcc-12'}/bin/gcc - cxx: {tmp_path / 'gcc-12'}/bin/g++ + c: {tmp_path / "gcc-12"}/bin/gcc + cxx: {tmp_path / "gcc-12"}/bin/g++ - spec: gcc@14.1.0 languages:=fortran - prefix: {tmp_path / 'gcc-14'} + prefix: {tmp_path / "gcc-14"} extra_attributes: compilers: - fortran: {tmp_path / 'gcc-14'}/bin/gfortran + fortran: {tmp_path / "gcc-14"}/bin/gfortran """ ) mutable_config.set("packages", packages_yaml["packages"]) @@ -3946,7 +3991,7 @@ def test_spec_parts_on_fresh_compilers( buildable: false externals: - spec: "llvm@20 +clang {constraint_in_yaml}" - prefix: {tmp_path / 'llvm-20'} + prefix: {tmp_path / "llvm-20"} """ ) mutable_config.set("packages", packages_yaml["packages"]) @@ -4009,7 +4054,7 @@ def test_spec_parts_on_reused_compilers( buildable: false externals: - spec: "llvm+clang@20 {constraint_in_yaml}" - prefix: {tmp_path / 'llvm-20'} + prefix: {tmp_path / "llvm-20"} mpileaks: buildable: true """ @@ -4420,7 +4465,10 @@ def _ensure_cache_hits(self, problem: str): # ensure subsequent concretizations of the same spec produce the same spec # object for _ in range(5): - assert h == spack.concretize.concretize_one("hdf5") + hdf5 = spack.concretize.concretize_one("hdf5") + + assert h.to_json(pretty=True) == hdf5.to_json(pretty=True) + assert h == hdf5 def test_concretization_cache_roundtrip_result(use_concretization_cache): @@ -4810,6 +4858,23 @@ def _mock_libc(self): assert mpileaks.satisfies("%c=gcc@12") +def test_concrete_specs_skip_prechecks(mock_packages): + """Test that concrete specs are not checked for unknown versions and dependencies.""" + + specs = [spack.spec.Spec("zlib"), spack.spec.Spec("deprecated-versions@=1.1.0")] + + with pytest.raises(spack.solver.asp.DeprecatedVersionError): + spack.solver.asp.SpackSolverSetup().setup(specs) + + with spack.config.override("config:deprecated", True): + concrete_spec = spack.concretize.concretize_one(specs[1]) + + # Try again with the same version but a concrete spec + specs[1] = concrete_spec + + spack.solver.asp.SpackSolverSetup().setup(specs) + + @pytest.mark.regression("51683") def test_activating_variant_for_conditional_language_dependency(default_mock_concretization): """Tests that a dependency on a conditional language can be concretized, and that the solver @@ -4824,8 +4889,29 @@ def test_activating_variant_for_conditional_language_dependency(default_mock_con assert s.satisfies("+fortran") +def test_when_condition_with_direct_dependency_on_virtual_provider(default_mock_concretization): + """If a when condition contains a direct dependency on a provider of a virtual, it should only + trigger if the provider is used for that current package, and not if the provider happens to be + a dependency, without its virtual being depended on.""" + s = default_mock_concretization("direct-dep-virtuals-one") + assert s.satisfies("%netlib-blas") + assert s["direct-dep-virtuals-two"].satisfies("%blas=netlib-blas") + + +def test_conflict_with_direct_dependency_on_virtual_provider(default_mock_concretization): + """Test that conflicts on virtual providers as direct dependencies work""" + s = default_mock_concretization("conflict-virtual") + assert s.satisfies("%blas=netlib-blas") + + with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): + default_mock_concretization("conflict-virtual +conflict_direct") + + with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): + default_mock_concretization("conflict-virtual +conflict_transitive") + + def test_imposed_spec_dependency_duplication(mock_packages: spack.repo.Repo): - """Tests that imposed dependenies triggered by identical conditions are grouped together, + """Tests that imposed dependencies triggered by identical conditions are grouped together, and that imposed dependencies that differ on a deptype are not grouped together.""" # The trigger-and-effect-deps pkg has 4 conditions, 2 triggers, and 4 effects in total: # +x -> depends on pkg-a with deptype link @@ -4846,3 +4932,161 @@ def test_imposed_spec_dependency_duplication(mock_packages: spack.repo.Repo): assert len([line for line in asp if re.search(r"trigger_id\(\d+\)", line)]) == 2 # There should be 4 effects total assert len([line for line in asp if re.search(r"effect_id\(\d+\)", line)]) == 4 + + +@pytest.mark.regression("51842") +@pytest.mark.parametrize( + "spec_str,expected", + [ + ("variant-function-validator", "generator=make %adios2~bzip2"), + ("variant-function-validator generator=make", "generator=make %adios2~bzip2"), + ("variant-function-validator generator=ninja", "generator=ninja %adios2+bzip2"), + ("variant-function-validator generator=other", "generator=other %adios2+bzip2"), + ], +) +def test_penalties_for_variant_defined_by_function( + default_mock_concretization, spec_str, expected +): + """Tests that we have penalties for variants defined by functions, and that variant values + are consistent with defaults and optimization rules. + """ + s = default_mock_concretization(spec_str) + assert s.satisfies(expected) + + +def test_default_values_used_if_subset_required_by_dependent(mock_packages): + """If a dependent requires *at least* a subset of default values of a multi-valued variant of + a dependency, that should not influence concretization; the default values should be used.""" + # multivalue-variant-multi-defaults-dependent requires myvariant=bar without baz. + a = spack.concretize.concretize_one("multivalue-variant-multi-defaults-dependent") + # we still end up using baz, and we don't drop it to avoid an extra dependency. + assert a.satisfies("%multivalue-variant-multi-defaults myvariant=bar,baz") + + +def test_virtual_gets_multiple_dupes(mock_packages, config): + """Tests that virtual packages always get multiple dupes, according to what we have in + the configuration files. + """ + specs = [spack.spec.Spec("pkg-with-c-link-dep")] + possible_graph = spack.solver.input_analysis.NoStaticAnalysis( + configuration=spack.config.CONFIG, repo=spack.repo.PATH + ) + counter = spack.solver.input_analysis.MinimalDuplicatesCounter( + specs, tests=False, possible_graph=possible_graph + ) + gen = spack.solver.asp.ProblemInstanceBuilder() + counter.possible_packages_facts(gen, spack.solver.core.fn) + + asp = gen.asp_problem + # "c" is a compiler language virtual and must allow multiple nodes, not be capped at 1 + selected_lines = [line for line in asp if line.startswith('max_dupes("c"')] + assert len(selected_lines) == 1 + max_dupes_c = selected_lines[0] + assert 'max_dupes("c",2).' == max_dupes_c, f"should have max_dupes=2, but got: {max_dupes_c}" + + +def test_compiler_selection_when_external_has_variant_penalty(mutable_config, mock_packages): + """Tests that a compiler that should be preferred is not swapped with a less preferred + compiler because of penalties on variants. + """ + packages_yaml = syaml.load_config( + """ +packages: + gcc:: + externals: + - spec: "gcc@15.2.0 languages='c,c++' ~binutils" + prefix: /path + extra_attributes: + compilers: + c: /path/bin/gcc + cxx: /path/bin/g++ + llvm:: + buildable: false + externals: + - spec: "llvm@20 +clang" + prefix: /path + extra_attributes: + compilers: + c: /path/bin/gcc + cxx: /path/bin/g++ +""" + ) + mutable_config.set("packages", packages_yaml["packages"]) + + concrete = spack.concretize.concretize_one("libdwarf") + + # GCC is the preferred provider, but has a penalty on its variants + assert concrete.satisfies("%gcc@15.2.0 ~binutils"), concrete.tree() + # LLVM is the second provider choice, with no penalty on variants + assert not concrete.satisfies("%llvm@20 +clang") + + +def test_mpi_selection_when_external_has_variant_penalty(mutable_config, mock_packages): + """Tests that conflicting with a default provider doesn't cause a variant values to be + flipped to avoid the variant dependency. + """ + packages_yaml = syaml.load_config( + """ +packages: + all: + variants: +mpi + mpich: + buildable: false +""" + ) + mutable_config.set("packages", packages_yaml["packages"]) + + concrete = spack.concretize.concretize_one("transitive-conditional-virtual-dependency") + + # GCC is the preferred provider, but has a penalty on its variants + assert concrete.satisfies("%conditional-virtual-dependency+mpi"), concrete.tree() + # LLVM is the second provider choice, with no penalty on variants + assert concrete.satisfies("^mpi=zmpi") + + +def test_preferring_different_compilers_for_different_languages(mutable_config, mock_packages): + """Tests that in a case where we prefer different compilers for different languages, steering + towards using a unique toolchain is lower priority with respect to flipping variants to turn + off a language, or selecting a non-default provider. + """ + packages_yaml = syaml.load_config( + """ +packages: + all: + providers: + c:: [llvm, gcc] + cxx:: [llvm, gcc] + fortran:: [gcc] + c: + prefer: + - llvm + cxx: + prefer: + - llvm + fortran: + prefer: + - gcc + mpileaks: + variants: +fortran +""" + ) + mutable_config.set("packages", packages_yaml["packages"]) + + mpileaks = spack.concretize.concretize_one("mpileaks") + + assert mpileaks.satisfies("%c,cxx=llvm %fortran=gcc"), mpileaks.tree() + assert mpileaks.satisfies("%mpi=mpich") + assert mpileaks["mpich"].satisfies("%c,cxx=llvm %fortran=gcc") + + +def test_specs_from_mirror_warns_when_index_missing(monkeypatch): + """Tests that we get a warning when a binary mirror has no index.""" + + def fake_update_cache(): + spack.binary_distribution.BINARY_INDEX.mirrors_without_index = {"file:///fake-mirror"} + return [] + + monkeypatch.setattr(spack.binary_distribution, "update_cache_and_get_specs", fake_update_cache) + + with pytest.warns(UserWarning, match="cannot be used in concretization"): + spack.solver.reuse._specs_from_mirror() diff --git a/lib/spack/spack/test/concretization/errors.py b/lib/spack/spack/test/concretization/errors.py index 017fd8e6e46954..ab25523c049dc8 100644 --- a/lib/spack/spack/test/concretization/errors.py +++ b/lib/spack/spack/test/concretization/errors.py @@ -1,14 +1,24 @@ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Regression tests for concretizer error messages. + +Every test asserts two properties: +1. The correct exception type is raised. +2. The message contains every "actionable part" -- a string from the user's + input (spec token, config key, package name) that helps identify what to + change. +""" import pathlib from io import StringIO +from typing import List import pytest import spack.concretize import spack.config +import spack.error import spack.main import spack.solver.asp import spack.spec @@ -106,3 +116,182 @@ def test_internal_error_handling_formatting(tmp_path: pathlib.Path): assert spack.spec.Spec.from_specfile(files["input-2.json"]) == spack.spec.Spec("bar+y") assert spack.spec.Spec.from_specfile(files["output-1.json"]) == spack.spec.Spec("foo@=1.0~x") assert spack.spec.Spec.from_specfile(files["output-2.json"]) == spack.spec.Spec("x@=1.0~y") + + +def assert_actionable_error(exc_info, *required_part: str) -> None: + """Verify that the error message contains every required part, which is usually a string that + the user can recognize in their own input. + """ + msg = str(exc_info.value) + missing = [h for h in required_part if h not in msg] + assert not missing, f"Error message is missing parts {missing!r}\nFull message:\n{msg}" + + +@pytest.mark.parametrize( + "input_spec,expected_parts", + [ + # fftw is constrained to ~mpi by the explicit request, but quantum-espresso + # requires fftw+mpi when +invino. Both values cannot coexist. + pytest.param( + "quantum-espresso+invino^fftw~mpi", ["fftw", "mpi"], id="variant_value_conflict" + ), + # The user requests a variant that does not exist on the package. + pytest.param( + "quantum-espresso+nonexistent", + ["quantum-espresso", "nonexistent", "No such variant"], + id="variant_undefined", + ), + # quantum-espresso has only version 1.0; @:0.1 cannot be satisfied. + pytest.param( + "quantum-espresso@:0.1", + ["quantum-espresso@:0.1", "No version exists"], + id="version_constraint_unsatisfied", + ), + # hypre propagates ~~shared to its deps, but openblas is explicitly +shared. + pytest.param( + "hypre ~~shared ^openblas +shared", + ["shared", "hypre", "'openblas' requires conflicting variant values"], + id="propagation_excluded", + ), + # dependency-foo-bar (++bar) and direct-dep-foo-bar (~~bar) both propagate + # variant "bar" with different values to their shared transitive dependency. + pytest.param( + "parent-foo-bar ^dependency-foo-bar++bar ^direct-dep-foo-bar~~bar", + ["cannot both propagate variant 'bar'"], + id="propagation_conflict_to_dep", + ), + # gmake is a build dependency of a transitive dep, not directly reachable + # via link/run from multivalue-variant. + pytest.param( + "multivalue-variant ^gmake", + ["gmake is not a direct 'build' or"], + id="literal_not_in_dag", + ), + # mvapich2 file_systems uses auto_or_any_combination_of, but "auto" and "lustre" + # come from disjoint sets and cannot be combined. + pytest.param( + "mvapich2 file_systems=auto,lustre", + ["mvapich2", "file_systems", "the value 'auto' is mutually exclusive"], + id="variant_disjoint_sets", + ), + # "fortan" is not a known virtual (typo of "fortran"). The error must name the + # unknown virtual and quote the originating spec, and must not be a generic internal error. + pytest.param( + "zlib %c,cxx,fortan=gcc", + ["fortan", "zlib %c,cxx,fortan=gcc", "not a known virtual"], + id="unknown_virtual_on_edge", + ), + # Two unknown virtuals on the same edge: both must appear in the single error raised. + pytest.param( + "zlib %c,fortan,cxxxx=gcc", + ["fortan", "cxxxx", "zlib %c,cxxxx,fortan=gcc"], + id="two_unknown_virtuals_on_edge", + ), + ], +) +def test_input_spec_driven_errors( + input_spec: str, expected_parts: List[str], mock_packages, mutable_config +) -> None: + """Tests errors caused by a token in the CLI input spec. The message must name both the + affected package and the specific token (variant, version, flag, dep) the user supplied. + """ + with pytest.raises(spack.error.SpackError) as exc_info: + spack.concretize.concretize_one(input_spec) + assert_actionable_error(exc_info, *expected_parts) + + +@pytest.mark.parametrize( + "packages_config,input_spec,expected_parts", + [ + # quantum-espresso is set buildable:false; the available external does not + # satisfy +veritas, so no valid spec can be found. + pytest.param( + { + "packages:quantum-espresso": { + "buildable": False, + "externals": [ + {"spec": "quantum-espresso@1.0~veritas", "prefix": "/path/to/qe"} + ], + } + }, + "quantum-espresso+veritas", + ["quantum-espresso", "it is configured `buildable:false`"], + id="buildable_false", + ), + # The user provided a packages.yaml `require:` with a message field. The error must surface + # the custom message so the user knows the policy and the package name so they can find + # the config section. + pytest.param( + { + "packages:libelf": { + "require": [{"spec": "%clang", "message": "must be compiled with clang"}] + } + }, + "libelf%gcc", + ["libelf", "must be compiled with clang"], + id="requirement_unsatisfied_custom_message", + ), + # Generic message must still name the package so the user knows which entry to look at + pytest.param( + {"packages:libelf": {"require": ["%clang"]}}, + "libelf%gcc", + ["libelf"], + id="requirement_unsatisfied_generic", + ), + # A `require:` entry names a virtual that does not exist. The error must name the + # unknown virtual and quote the originating spec so the user can find and fix the + # config entry. + pytest.param( + {"packages:zlib": {"require": ["%[virtuals=fortan]gcc"]}}, + "zlib", + ["fortan", "%[virtuals=fortan]gcc"], + id="unknown_virtual_in_requirement", + ), + # Two unknown virtuals in a single requirement spec: both must appear in the error. + pytest.param( + {"packages:zlib": {"require": ["%[virtuals=fortan,cxxxx]gcc"]}}, + "zlib", + ["fortan", "cxxxx", "%[virtuals=fortan,cxxxx]gcc"], + id="two_unknown_virtuals_in_requirement", + ), + ], +) +def test_config_driven_errors( + packages_config, input_spec: str, expected_parts: List[str], mock_packages, mutable_config +) -> None: + """Tests errors caused by user configuration, e,g, a setting in packages.yaml. The message must + identify the package and the config value to fix. + """ + for path, conf in packages_config.items(): + spack.config.set(path, conf) + + with pytest.raises(spack.error.SpackError) as exc_info: + spack.concretize.concretize_one(input_spec) + assert_actionable_error(exc_info, *expected_parts) + + +@pytest.mark.parametrize( + "input_spec,expected_handles", + [ + # conflict-parent@0.9 has conflicts("^conflict~foo", when="@0.9"). When the user requests + # `^conflict~foo` the conflict fires. The auto-generated message includes the package name + # and the when-spec version, giving the user two places to look. + pytest.param( + "conflict-parent@0.9 ^conflict~foo", + ["conflict-parent", "'^conflict~foo' conflicts with '@0.9'"], + id="conflicts_directive", + ), + # requires-clang has `requires("%clang", msg="can only be compiled with Clang")`. When + # compiled with %gcc the requirement is unsatisfied and the custom message is shown + pytest.param("requires-clang %gcc", ["requires-clang", "Clang"], id="requires_directive"), + ], +) +def test_package_py_driven_errors( + input_spec: str, expected_handles: List[str], mock_packages, mutable_config +) -> None: + """Tests errors involving directives in package.py recipes. The error message must name the + package whose directive caused the failure. + """ + with pytest.raises(spack.error.SpackError) as exc_info: + spack.concretize.concretize_one(input_spec) + assert_actionable_error(exc_info, *expected_handles) diff --git a/lib/spack/spack/test/concretization/requirements.py b/lib/spack/spack/test/concretization/requirements.py index 76e4d5a242dff7..140e77b69b7595 100644 --- a/lib/spack/spack/test/concretization/requirements.py +++ b/lib/spack/spack/test/concretization/requirements.py @@ -20,7 +20,7 @@ import spack.version from spack.installer import PackageInstaller from spack.solver.asp import InternalConcretizerError, UnsatisfiableSpecError -from spack.solver.reuse import SpecFilter, create_external_parser +from spack.solver.reuse import create_external_parser, spec_filter_from_packages_yaml from spack.solver.runtimes import external_config_with_implicit_externals from spack.spec import Spec from spack.util.url import path_to_file_url @@ -159,9 +159,7 @@ def test_requirement_adds_new_version( packages: v: require: "@{0}=2.2" -""".format( - a_commit_hash - ) +""".format(a_commit_hash) update_packages_config(conf_str) s1 = spack.concretize.concretize_one("v") @@ -192,9 +190,7 @@ def test_requirement_adds_version_satisfies( packages: t: require: "@{0}=2.2" -""".format( - commits[0] - ) +""".format(commits[0]) update_packages_config(conf_str) s1 = spack.concretize.concretize_one("t") @@ -312,7 +308,7 @@ def test_requirement_is_successfully_applied(concretize_scope, test_repo): """ update_packages_config(conf_str) s2 = spack.concretize.concretize_one("x") - # The requirement forces choosing the eariler version + # The requirement forces choosing the earlier version assert s2.satisfies("@1.0") @@ -1323,7 +1319,7 @@ def test_requirements_on_compilers_and_reuse( root_specs = [Spec(input_spec)] packages_with_externals = external_config_with_implicit_externals(mutable_config) completion_mode = mutable_config.get("concretizer:externals:completion") - external_specs = SpecFilter.from_packages_yaml( + external_specs = spec_filter_from_packages_yaml( external_parser=create_external_parser(packages_with_externals, completion_mode), packages_with_externals=packages_with_externals, include=[], @@ -1513,7 +1509,7 @@ def test_language_preferences_and_reuse( reused_nodes = list(initial_mpileaks.traverse()) packages_with_externals = external_config_with_implicit_externals(mutable_config) completion_mode = mutable_config.get("concretizer:externals:completion") - external_specs = SpecFilter.from_packages_yaml( + external_specs = spec_filter_from_packages_yaml( external_parser=create_external_parser(packages_with_externals, completion_mode), packages_with_externals=packages_with_externals, include=[], @@ -1647,3 +1643,23 @@ def test_penalties_for_language_preferences(concretize_scope, mock_packages): assert s.satisfies("%c=gcc@10") assert all(s[name].satisfies("%c=clang") for name in dependency_names) assert s["mpi"].satisfies("%c,cxx=clang %fortran=gcc@10") + + +def test_prefer_when_condition_expands_toolchain(concretize_scope, mutable_config, mock_packages): + """Tests that toolchains in the 'when' condition of a 'prefer' rule must are expanded.""" + # If the expansion to %gcc doesn't happen, the preference for @2.1 is silently ignored + mutable_config.set("toolchains", {"gcc_toolchain": "%c=gcc"}, scope="concretize") + update_packages_config(""" +packages: + multivalue-variant: + prefer: + - spec: "@2.1" + when: "%gcc_toolchain" +""") + + s_gcc = spack.concretize.concretize_one("multivalue-variant %c=gcc") + assert s_gcc.satisfies("@2.1 %c=gcc"), f"expected @2.1 with gcc, got {s_gcc.version}" + + # With clang as compiler, condition does not fire -> default highest version @2.3 + s_clang = spack.concretize.concretize_one("multivalue-variant %clang") + assert s_clang.satisfies("@2.3 %c=clang"), f"expected @2.3 with clang, got {s_clang.version}" diff --git a/lib/spack/spack/test/concretization/splicing.py b/lib/spack/spack/test/concretization/splicing.py index 0b331d89011fcb..f93b20b74f7d65 100644 --- a/lib/spack/spack/test/concretization/splicing.py +++ b/lib/spack/spack/test/concretization/splicing.py @@ -22,13 +22,7 @@ def _make_specs_non_buildable(specs: List[str]): @pytest.fixture -def install_specs( - mutable_database, - mock_packages, - mutable_config, - do_not_check_runtimes_on_reuse, - install_mockery, -): +def install_specs(mutable_database, mock_packages, mutable_config, install_mockery): """Returns a function that concretizes and installs a list of abstract specs""" mutable_config.set("concretizer:reuse", True) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 68fd5d9fb20e9c..9a160439b0f763 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1180,7 +1180,7 @@ def test_single_file_scope_cache_clearing(env_yaml): assert before # Clear the cache of the Single file scope scope.clear() - # Check that the section can be retireved again and it's + # Check that the section can be retrieved again and it's # the same as before after = scope.get_section("config") assert after @@ -1265,7 +1265,7 @@ def mock_include_scope(tmp_path): @pytest.fixture def include_config_factory(mock_include_scope): def make_config(): - cfg = spack.config.create() + cfg = spack.config.Configuration() cfg.push_scope( spack.config.DirectoryConfigScope("defaults", str(mock_include_scope / "defaults")), priority=ConfigScopePriority.DEFAULTS, @@ -1390,6 +1390,8 @@ def test_override_included_config(working_env, tmp_path, include_config_factory) include_yaml = override_scope / "include.yaml" subdir = override_scope / "subdir" subdir.mkdir() + anotherdir = override_scope / "anotherdir" + anotherdir.mkdir() with include_yaml.open("w", encoding="utf-8") as f: f.write( @@ -1402,20 +1404,40 @@ def test_override_included_config(working_env, tmp_path, include_config_factory) ) ) + with (subdir / "include.yaml").open("w", encoding="utf-8") as f: + f.write( + textwrap.dedent( + """\ + include: + - name: "anotherdir" + path: "../anotherdir" + """ + ) + ) + # check the mock config is correct cfg = include_config_factory() assert "defaults" in cfg.scopes + assert "tmp_path" in cfg.scopes assert "test1" in cfg.scopes assert "test2" in cfg.scopes assert "test3" in cfg.scopes active_names = [s.name for s in cfg.active_scopes] assert "defaults" in active_names + assert "tmp_path" in active_names assert "test1" in active_names assert "test2" in active_names assert "test3" in active_names + includes = str(cfg.get("include")) + assert "subdir" not in includes + assert "anotherdir" not in includes + assert "test1" in includes + assert "test2" in includes + assert "test3" in includes + # push a scope that overrides everything under it but includes a subdir. # its included subdir should be active, but scopes *not* included by the overriding # scope should not. @@ -1425,34 +1447,54 @@ def test_override_included_config(working_env, tmp_path, include_config_factory) ) assert "defaults" in cfg.scopes + assert "tmp_path" in cfg.scopes assert "test1" in cfg.scopes assert "test2" in cfg.scopes assert "test3" in cfg.scopes assert "override" in cfg.scopes assert "subdir" in cfg.scopes + assert "anotherdir" in cfg.scopes active_names = [s.name for s in cfg.active_scopes] assert "defaults" in active_names + assert "tmp_path" in active_names assert "test1" not in active_names assert "test2" not in active_names assert "test3" not in active_names assert "override" in active_names assert "subdir" in active_names + assert "anotherdir" not in active_names + + includes = str(cfg.get("include")) + assert "subdir" in includes + assert "anotherdir" not in includes + assert "test1" not in includes + assert "test2" not in includes + assert "test3" not in includes # remove the override and ensure everything is back to normal cfg.remove_scope("override") assert "defaults" in cfg.scopes + assert "tmp_path" in cfg.scopes assert "test1" in cfg.scopes assert "test2" in cfg.scopes assert "test3" in cfg.scopes active_names = [s.name for s in cfg.active_scopes] assert "defaults" in active_names + assert "tmp_path" in active_names assert "test1" in active_names assert "test2" in active_names assert "test3" in active_names + includes = str(cfg.get("include")) + assert "subdir" not in includes + assert "anotherdir" not in includes + assert "test1" in includes + assert "test2" in includes + assert "test3" in includes + def test_user_cache_path_is_overridable(working_env): p = "/some/path" @@ -1484,7 +1526,7 @@ def test_config_file_read_perms_failure(tmp_path: pathlib.Path, mutable_empty_co def test_config_file_read_invalid_yaml(tmp_path: pathlib.Path, mutable_empty_config): - """Test reading a configuration file with invalid (unparseable) YAML + """Test reading a configuration file with invalid (unparsable) YAML raises a ConfigFileError.""" filename = join_path(str(tmp_path), "test.yaml") with open(filename, "w", encoding="utf-8") as f: @@ -1685,6 +1727,20 @@ def test_included_path_string_no_parent_path( assert curr_dir == os.path.commonprefix([curr_dir, destination]) # type: ignore[list-item] +def test_included_path_substitution(): + # check a straight path substitution + entry = {"path": "$user_cache_path/path/to/config.yaml"} + include = spack.config.included_path(entry) + assert spack.paths.user_cache_path in include.path + + # check path through an environment variable + path = "/path/to/project/packages.yaml" + os.environ["SPACK_TEST_PATH_SUB"] = path + entry = {"name": "vartest", "path": "$SPACK_TEST_PATH_SUB"} + include = spack.config.included_path(entry) + assert path in include.path + + def test_included_path_conditional_bad_when( tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, capfd ): @@ -1745,7 +1801,7 @@ def test_included_path_git_unsat( } include = spack.config.included_path(entry) assert isinstance(include, spack.config.GitIncludePaths) - assert include.repo == entry["git"] + assert include.git == entry["git"] assert include.tag == entry["tag"] assert include.paths == entry["paths"] assert include.when == entry["when"] @@ -1757,30 +1813,49 @@ def test_included_path_git_unsat( assert not scopes +def test_included_path_git_substitutions(): + # check path substitutions for the git url *and* paths + paths = ["./$platform/config.yaml", "$platform/packages.yaml"] + entry = { + "git": "https://example.com/$platform/configs.git", + "branch": "develop", + "name": "site", + "paths": paths, + "when": 'platform == "test"', + } + include = spack.config.included_path(entry) + assert isinstance(include, spack.config.GitIncludePaths) + assert not include.optional and include.evaluate_condition() + assert "test" in include.git, "Expected the git url to contain the platform" + for path in include.paths: + assert "test" in path, "Expected the included git path to contain the platform" + + # check environment substitution for the git url + url = "https://example.com/path/to/configs.git" + os.environ["SPACK_TEST_URL_SUB"] = url + entry["git"] = "$SPACK_TEST_URL_SUB" + include = spack.config.included_path(entry) + assert include.git == url, "Expected git url environment var substitution" + + @pytest.mark.parametrize( "key,value", [("branch", "main"), ("commit", "abcdef123456"), ("tag", "v1.0")] ) def test_included_path_git( tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, monkeypatch, key, value, capfd ): - monkeypatch.setattr(spack.paths, "user_cache_path", str(tmp_path)) - - class MockIncludeGit(spack.util.executable.Executable): - def __init__(self, required: bool): - pass - - def __call__(self, *args, **kwargs) -> str: # type: ignore - action = args[0] + """Check git includes for branch, commit, and tag using relative paths. - if action == "config": - return "origin" - - return "" + Note the mock config fixture does NOT create the scope path so a temporary + directory will be used for caching the files. + """ - paths = ["config.yaml", "packages.yaml"] + # Specifying two relative paths, one explicit, one implicit + paths = ["./config.yaml", "packages.yaml"] entry = { "git": "https://example.com/windows/configs.git", key: value, + "name": "site", "paths": paths, "when": 'platform == "test"', } @@ -1788,18 +1863,30 @@ def __call__(self, *args, **kwargs) -> str: # type: ignore assert isinstance(include, spack.config.GitIncludePaths) assert not include.optional and include.evaluate_condition() - destination = include._destination() - assert not os.path.exists(destination) - # set up minimal git and repository operations + class MockIncludeGit(spack.util.executable.Executable): + def __init__(self, required: bool): + pass + + def __call__(self, *args, **kwargs) -> str: # type: ignore + action = args[0] + + if action == "config": + return "origin" + + return "" + monkeypatch.setattr(spack.util.git, "git", MockIncludeGit) def _init_repo(*args, **kwargs): - fs.mkdirp(fs.join_path(destination, ".git")) + # Make sure the directory exists, where assuming called from within + # the working directory. + fs.mkdirp(fs.join_path(os.getcwd(), ".git")) def _checkout(*args, **kwargs): - # Make sure the files exist at the clone destination - with fs.working_dir(destination): + # Make sure the files exist at the clone destination, where assuming + # called from within the working directory. + with fs.working_dir(os.getcwd()): for p in paths: fs.touch(p) @@ -1809,10 +1896,13 @@ def _checkout(*args, **kwargs): # First successful pass builds the scope parent_scope = mock_low_high_config.scopes["low"] scopes = include.scopes(parent_scope) - assert scopes and len(scopes) == len(paths) + assert len(scopes) == len(paths) + + base_paths = [os.path.basename(p) for p in paths] for scope in scopes: assert isinstance(scope, spack.config.SingleFileScope) - assert os.path.basename(scope.path) in paths # type: ignore[union-attr] + assert os.path.basename(scope.path) in base_paths # type: ignore[union-attr] + assert scope.name.split(":")[1] in base_paths # Second pass uses the scopes previously built. # Only need to do this for one of the parameters. @@ -1825,11 +1915,62 @@ def _checkout(*args, **kwargs): # A direct clone now returns already cloned destination and debug message. # Again only need to run this test once. if key == "tag": - assert include._clone() == include.destination + assert include._clone(parent_scope) == include.destination captured = capfd.readouterr()[1] assert "already cloned" in captured +@pytest.mark.parametrize("path", ["./config.yaml", "/path/to/my/special/package.yaml"]) +def test_included_path_local_no_dest(path): + """Confirm that local paths have no cache destination.""" + entry = {"path": path} + include = spack.config.included_path(entry) + destination = include.base_directory(entry["path"]) + assert not destination, f"Expected local include ({include}) to NOT have a cache destination" + + +def test_included_path_url_temp_dest(mock_low_high_config): + """Check that remote (raw) path under different scopes end up with temporary + cache destinations.""" + entry = { + "path": "https://github.com/path/to/raw/config/config.yaml", + "sha256": "26e871804a92cd07bb3d611b31b4156ae93d35b6a6d6e0ef3a67871fcb1d258b", + } + include = spack.config.included_path(entry) + + parent_scope = mock_low_high_config.scopes["low"] + parent_scope.path = "" + pre = f"Expected temporary cache destination for raw include path ({include}) for " + + for scope in [None, parent_scope]: + rest = "parent scope with no path" if scope else "no parent scope" + destination = include.base_directory(entry["path"], parent_scope=scope) + dest_dir = str(pathlib.Path(destination).parent) + temp_dir = tempfile.gettempdir() + assert dest_dir == temp_dir, pre + rest + + +def test_included_path_git_temp_dest(mock_low_high_config): + """Check a remote (relative) path with different parent scope options that + result in a temporary cache destination.""" + entry = { + "git": "https://example.com/linux/configs.git", + "branch": "develop", + "paths": ["config.yaml"], + } + include = spack.config.included_path(entry) + parent_scope = mock_low_high_config.scopes["low"] + parent_scope.path = "" + pre = f"Expected temporary cache destination for git include path ({include}) for " + + for scope in [None, parent_scope]: + rest = "parent scope with no path" if scope else "no parent scope" + destination = include.base_directory(entry["git"], parent_scope=scope) + dest_dir = str(pathlib.Path(destination).parent) + temp_dir = tempfile.gettempdir() + assert dest_dir == temp_dir, pre + rest + + def test_included_path_git_errs(tmp_path: pathlib.Path, mock_low_high_config, monkeypatch): monkeypatch.setattr(spack.paths, "user_cache_path", str(tmp_path)) @@ -1878,9 +2019,9 @@ def __call__(self, *args, **kwargs) -> str: # type: ignore def test_missing_include_scope_list(mock_missing_dir_include_scopes): """Tests that an included scope with a non existent file/directory is still listed as a scope under spack.config.CONFIG.scopes""" - assert "sub_base" in list( - spack.config.CONFIG.scopes - ), "Missing Optional Scope Missing from Config Scopes" + assert "sub_base" in list(spack.config.CONFIG.scopes), ( + "Missing Optional Scope Missing from Config Scopes" + ) def test_missing_include_scope_writable_list(mock_missing_dir_include_scopes): @@ -1896,7 +2037,7 @@ def test_missing_include_scope_not_readable_list(mock_missing_dir_include_scopes def test_missing_include_scope_default_created_as_dir_scope(mock_missing_dir_include_scopes): - """Tests that an optional include with no existing file/directory and no yaml extention + """Tests that an optional include with no existing file/directory and no yaml extension is created as a directoryscope object""" missing_inc_scope = spack.config.CONFIG.scopes["sub_base"] assert isinstance(missing_inc_scope, spack.config.DirectoryConfigScope) @@ -1912,34 +2053,34 @@ def test_missing_include_scope_yaml_ext_is_file_scope(mock_missing_file_include_ def test_missing_include_scope_writeable_not_readable(mock_missing_dir_include_scopes): """Tests that an included scope with a non existent file/directory can be written to (and created)""" - assert spack.config.CONFIG.scopes[ - "sub_base" - ].writable, "Missing Optional Scope should be writable" - assert not spack.config.CONFIG.scopes[ - "sub_base" - ].exists, "Missing Optional Scope should not exist" + assert spack.config.CONFIG.scopes["sub_base"].writable, ( + "Missing Optional Scope should be writable" + ) + assert not spack.config.CONFIG.scopes["sub_base"].exists, ( + "Missing Optional Scope should not exist" + ) def test_missing_include_scope_empty_read(mock_missing_dir_include_scopes): """Tests that an included scope with a non existent file/directory returns an empty dict on read and has "exists" set to false""" - assert ( - spack.config.CONFIG.get("config", scope="sub_base") == {} - ), "Missing optional include scope does not return an empty value." - assert not spack.config.CONFIG.scopes[ - "sub_base" - ].exists, "Missing optional include should not be created on read" + assert spack.config.CONFIG.get("config", scope="sub_base") == {}, ( + "Missing optional include scope does not return an empty value." + ) + assert not spack.config.CONFIG.scopes["sub_base"].exists, ( + "Missing optional include should not be created on read" + ) def test_missing_include_scope_file_empty_read(mock_missing_file_include_scopes): """Tests that an include scope with a non existent file returns an empty dict and has exists set to false""" - assert ( - spack.config.CONFIG.get("config", scope="sub_base") == {} - ), "Missing optional include scope does not return an empty value." - assert not spack.config.CONFIG.scopes[ - "sub_base" - ].exists, "Missing optional include should not be created on read" + assert spack.config.CONFIG.get("config", scope="sub_base") == {}, ( + "Missing optional include scope does not return an empty value." + ) + assert not spack.config.CONFIG.scopes["sub_base"].exists, ( + "Missing optional include should not be created on read" + ) def test_missing_include_scope_write_directory(mock_missing_dir_include_scopes): @@ -1960,3 +2101,34 @@ def test_missing_include_scope_write_file(mock_missing_file_include_scopes): assert os.path.exists(spack.config.CONFIG.scopes["sub_base"].path) install_root = spack.config.CONFIG.get("config:install_tree:root", scope="sub_base") assert install_root == "$spack/tmp/spack" + + +def test_config_scope_empty_write(tmp_path: pathlib.Path): + """Confirm skipping attempt to write non-existent scope section.""" + config_scope = spack.config.DirectoryConfigScope("test", str(tmp_path)) + + assert config_scope.get_section("include") is None + + +def test_include_bad_parent_scope(tmp_path: pathlib.Path): + """Test parent scope validation.""" + path = tmp_path / "config.yaml" + path.touch() + entry = {"path": str(path)} + include = spack.config.included_path(entry) + + # Confirm require a ConfigScope parent + with pytest.raises(AssertionError, match="configuration scope"): + _ = include.scopes("_builtin") # type: ignore + + # Confirm require a named parent scope + for name in ["", " "]: + parent_scope = spack.config.InternalConfigScope(name, spack.config.CONFIG_DEFAULTS) + with pytest.raises(AssertionError, match="must have a name"): + _ = include.scopes(parent_scope) + + +def test_config_invalid_scope(mock_low_high_config): + err = "Must be one of \\['low', 'high'\\]" # noqa: W605 + with pytest.raises(ValueError, match=err): + spack.config.CONFIG.get_config_filename("noscope", "nosection") diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index bf7d7dd1a62ec0..126698b453627f 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -18,9 +18,10 @@ import stat import sys import tempfile +import textwrap import xml.etree.ElementTree from pathlib import Path -from typing import Callable, List, Optional, Tuple +from typing import Callable, List, Optional, Tuple, Union import pytest @@ -75,7 +76,6 @@ join_path, mkdirp, remove_linked_tree, - touchp, working_dir, ) from spack.main import SpackCommand @@ -459,7 +459,7 @@ def use_concretization_cache(mock_packages, mutable_config, tmp_path: Path): # @pytest.fixture(scope="function", autouse=True) def no_chdir(): - """Ensure that no test changes Spack's working dirctory. + """Ensure that no test changes Spack's working directory. This prevents Spack tests (and therefore Spack commands) from changing the working directory and causing other tests to fail @@ -477,7 +477,7 @@ def no_chdir(): def onerror(func, path, error_info): - # Python on Windows is unable to remvove paths without + # Python on Windows is unable to remove paths without # write (IWUSR) permissions (such as those generated by Git on Windows) # This method changes file permissions to allow removal by Python os.chmod(path, stat.S_IWUSR) @@ -632,7 +632,7 @@ def mock_binary_index(monkeypatch, tmp_path_factory: pytest.TempPathFactory): """ tmpdir = tmp_path_factory.mktemp("mock_binary_index") index_path = tmpdir / "binary_index" - mock_index = spack.binary_distribution.BinaryCacheIndex(str(index_path)) + mock_index = spack.binary_distribution.BinaryIndexCache(str(index_path)) monkeypatch.setattr(spack.binary_distribution, "BINARY_INDEX", mock_index) yield @@ -702,8 +702,6 @@ def mock_packages_repo(): def _pkg_install_fn(pkg, spec, prefix): # sanity_check_prefix requires something in the install directory mkdirp(prefix.bin) - if not os.path.exists(spec.package.install_log_path): - touchp(spec.package.install_log_path) @pytest.fixture @@ -936,6 +934,40 @@ def configuration_dir(tmp_path_factory: pytest.TempPathFactory, linux_os): modules = tmp_path / "site" / "modules.yaml" modules_template = test_config / "modules.yaml" modules.write_text(modules_template.read_text().format(tcl_root, lmod_root)) + + for scope in ("spack", "user", "site", "system"): + scope_path = tmp_path / scope + scope_path.mkdir(exist_ok=True) + + include = tmp_path / "spack" / "include.yaml" + # Need to use relative include paths here so it works for mutable_config fixture too + with include.open("w", encoding="utf-8") as f: + f.write( + textwrap.dedent( + """ + include: + # user configuration scope + - name: "user" + path_override_env_var: SPACK_USER_CONFIG_PATH + path: ../user + optional: true + prefer_modify: true + when: '"SPACK_DISABLE_LOCAL_CONFIG" not in env' + + # site configuration scope + - name: "site" + path: ../site + optional: true + + # system configuration scope + - name: "system" + path_override_env_var: SPACK_SYSTEM_CONFIG_PATH + path: ../system + optional: true + when: '"SPACK_DISABLE_LOCAL_CONFIG" not in env' + """ + ) + ) yield tmp_path @@ -948,15 +980,7 @@ def _create_mock_configuration_scopes(configuration_dir): ), ( ConfigScopePriority.CONFIG_FILES, - spack.config.DirectoryConfigScope("site", str(configuration_dir / "site")), - ), - ( - ConfigScopePriority.CONFIG_FILES, - spack.config.DirectoryConfigScope("system", str(configuration_dir / "system")), - ), - ( - ConfigScopePriority.CONFIG_FILES, - spack.config.DirectoryConfigScope("user", str(configuration_dir / "user")), + spack.config.DirectoryConfigScope("spack", str(configuration_dir / "spack")), ), (ConfigScopePriority.COMMAND_LINE, spack.config.InternalConfigScope("command_line")), ] @@ -1078,7 +1102,7 @@ def create_config_scope(path: Path, name: str) -> spack.config.DirectoryConfigSc @pytest.fixture() def mock_missing_dir_include_scopes(tmp_path: Path): """Mocks a config scope containing optional directory scope - includes that do not have represetation on the filesystem""" + includes that do not have representation on the filesystem""" scope = create_config_scope(tmp_path, "sub") with spack.config.use_configuration(scope) as config: @@ -1088,7 +1112,7 @@ def mock_missing_dir_include_scopes(tmp_path: Path): @pytest.fixture def mock_missing_file_include_scopes(tmp_path: Path): """Mocks a config scope containing optional file scope - includes that do not have represetation on the filesystem""" + includes that do not have representation on the filesystem""" scope = create_config_scope(tmp_path, "sub.yaml") with spack.config.use_configuration(scope) as config: @@ -1512,7 +1536,7 @@ def get_cvs_timestamp(output): # We use this to record the time stamps for when we create CVS revisions, # so that we can later check that we retrieve the proper commits when - # specifying a date. (CVS guarantees checking out the lastest revision + # specifying a date. (CVS guarantees checking out the latest revision # before or on the specified date). As we create each revision, we # separately record the time by querying CVS. revision_date = {} @@ -1756,7 +1780,9 @@ def mock_git_repository(git, tmp_path_factory: pytest.TempPathFactory): revision=tag_branch, file=tag_file, args={"git": url, "branch": tag_branch} ), "tag": Bunch(revision=tag, file=tag_file, args={"git": url, "tag": tag}), - "commit": Bunch(revision=r1, file=r1_file, args={"git": url, "commit": r1}), + "commit": Bunch( + revision=r1, file=r1_file, args={"git": url, "branch": branch, "commit": r1} + ), "annotated-tag": Bunch(revision=a_tag, file=r2_file, args={"git": url, "tag": a_tag}), # In this case, the version() args do not include a 'git' key: # this is the norm for packages, so this tests how the fetching logic @@ -2010,13 +2036,13 @@ def mock_directive_bundle(): @pytest.fixture def clear_directive_functions(): - """Clear all overidden directive functions for subsequent tests.""" + """Clear all overridden directive functions for subsequent tests.""" yield - # Make sure any directive functions overidden by tests are cleared before + # Make sure any directive functions overridden by tests are cleared before # proceeding with subsequent tests that may depend on the original # functions. - spack.directives_meta.DirectiveMeta._directives_to_be_executed = [] + spack.directives_meta.DirectiveMeta._directives_to_be_executed.clear() @pytest.fixture @@ -2067,7 +2093,7 @@ def inode_cache(): def brand_new_binary_cache(): yield spack.binary_distribution.BINARY_INDEX = spack.llnl.util.lang.Singleton( - spack.binary_distribution.BinaryCacheIndex + spack.binary_distribution.BinaryIndexCache ) @@ -2230,9 +2256,7 @@ def _factory(rpaths, message="Hello world!", dynamic_linker="/lib64/ld-linux.so. int main(){{ printf("{0}"); }} - """.format( - message - ) + """.format(message) ) gcc = spack.util.executable.which("gcc", required=True) executable = source.parent / "main.x" @@ -2459,12 +2483,6 @@ def _include_cache_root(): return join_path(str(tempfile.mkdtemp()), "user_cache", "includes") -@pytest.fixture() -def mock_include_cache(monkeypatch): - """Override the include cache directory so tests don't pollute user cache.""" - monkeypatch.setattr(spack.config, "_include_cache_location", _include_cache_root) - - @pytest.fixture() def wrapper_dir(install_mockery): """Installs the compiler wrapper and returns the prefix where the script is installed.""" @@ -2553,3 +2571,43 @@ def reset_extension_paths(): spack.extensions.extension_paths_from_entry_points.cache_clear() yield spack.extensions.extension_paths_from_entry_points.cache_clear() + + +@pytest.fixture(params=["old", "new"]) +def installer_variant(request): + """Parametrize a test over the old and new installer.""" + if request.param == "new" and sys.platform == "win32": + pytest.skip("New installer not supported on Windows") + with spack.config.override("config:installer", request.param): + yield request.param + + +class FsTree: + class symlink: + def __init__(self, target): + self.target = target + + class file: + def __init__(self, content: Union[bytes, str] = b""): + self.content = content + + class dir: + pass + + def __init__(self, base_path: Path, layout: dict): + for rel_path, content in layout.items(): + p = base_path / rel_path + p.parent.mkdir(parents=True, exist_ok=True) + + assert isinstance(content, (self.symlink, self.file, self.dir)) + + if isinstance(content, self.dir): + p.mkdir(exist_ok=True) + elif isinstance(content, self.symlink): + p.symlink_to(content.target) + elif isinstance(content, self.file): + assert isinstance(content.content, (bytes, str)) + if isinstance(content.content, bytes): + p.write_bytes(content.content) + elif isinstance(content.content, str): + p.write_text(content.content, encoding="utf-8") diff --git a/lib/spack/spack/test/cray_manifest.py b/lib/spack/spack/test/cray_manifest.py index 61ec79fbcd6bbc..865537cb2a11bb 100644 --- a/lib/spack/spack/test/cray_manifest.py +++ b/lib/spack/spack/test/cray_manifest.py @@ -8,6 +8,7 @@ establish dependency relationships (and in general the manifest-parsing logic needs to consume all related specs in a single pass). """ + import json import pathlib diff --git a/lib/spack/spack/test/data/config/concretizer.yaml b/lib/spack/spack/test/data/config/concretizer.yaml index a1a30ff0280bde..382c694c3de5de 100644 --- a/lib/spack/spack/test/data/config/concretizer.yaml +++ b/lib/spack/spack/test/data/config/concretizer.yaml @@ -3,7 +3,30 @@ concretizer: targets: granularity: microarchitectures host_compatible: false + duplicates: strategy: minimal + max_dupes: + default: 1 + # Virtuals + c: 2 + cxx: 2 + fortran: 1 + # Regular packages + cmake: 2 + gmake: 2 + python: 2 + python-venv: 2 + py-cython: 2 + py-flit-core: 2 + py-pip: 2 + py-setuptools: 2 + py-versioneer: 2 + py-wheel: 2 + xcb-proto: 2 + # Compilers + gcc: 2 + llvm: 2 + concretization_cache: enable: false diff --git a/lib/spack/spack/test/data/config/config.yaml b/lib/spack/spack/test/data/config/config.yaml index e6867adb3db9b2..20bcd2a7e1af88 100644 --- a/lib/spack/spack/test/data/config/config.yaml +++ b/lib/spack/spack/test/data/config/config.yaml @@ -12,5 +12,6 @@ config: verify_ssl: true ssl_certs: $SSL_CERT_FILE checksum: true + installer: old # many tests are based on stdout from old installer dirty: false locks: {1} diff --git a/lib/spack/spack/test/data/directory_search/README.txt b/lib/spack/spack/test/data/directory_search/README.txt index 9c43a4224d7108..abef744b44feda 100644 --- a/lib/spack/spack/test/data/directory_search/README.txt +++ b/lib/spack/spack/test/data/directory_search/README.txt @@ -1 +1 @@ -This directory tree is made up to test that search functions wil return a stable ordered sequence. \ No newline at end of file +This directory tree is made up to test that search functions will return a stable ordered sequence. \ No newline at end of file diff --git a/lib/spack/spack/test/data/modules/lmod/complex_hierarchy.yaml b/lib/spack/spack/test/data/modules/lmod/complex_hierarchy.yaml index 91adfd92e63e73..a408d3ee0a238c 100644 --- a/lib/spack/spack/test/data/modules/lmod/complex_hierarchy.yaml +++ b/lib/spack/spack/test/data/modules/lmod/complex_hierarchy.yaml @@ -13,6 +13,7 @@ lmod: - lapack - blas - mpi + - python filter_hierarchy_specs: 'mpileaks@:2.1': [mpi] diff --git a/lib/spack/spack/test/data/modules/lmod/non_virtual_in_hierarchy.yaml b/lib/spack/spack/test/data/modules/lmod/non_virtual_in_hierarchy.yaml deleted file mode 100644 index bf9df440d03843..00000000000000 --- a/lib/spack/spack/test/data/modules/lmod/non_virtual_in_hierarchy.yaml +++ /dev/null @@ -1,11 +0,0 @@ -enable: - - lmod -lmod: - core_compilers: - - 'clang@3.3' - hierarchy: - - mpi - - openblas - - all: - autoload: direct diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 7e208f6fb736b0..480b538412bed2 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Check the database is functioning properly, both in memory and in its file.""" + import contextlib import datetime import functools @@ -352,11 +353,13 @@ def test_recursive_upstream_dbs( ) assert db_a_from_scratch.db_for_spec_hash(spec.dag_hash()) == (db_a_from_scratch) - assert db_a_from_scratch.db_for_spec_hash(spec["y"].dag_hash()) == ( - upstream_dbs_from_scratch[0] + assert ( + db_a_from_scratch.db_for_spec_hash(spec["y"].dag_hash()) + == (upstream_dbs_from_scratch[0]) ) - assert db_a_from_scratch.db_for_spec_hash(spec["z"].dag_hash()) == ( - upstream_dbs_from_scratch[1] + assert ( + db_a_from_scratch.db_for_spec_hash(spec["z"].dag_hash()) + == (upstream_dbs_from_scratch[1]) ) db_a_from_scratch._check_ref_counts() @@ -592,6 +595,22 @@ def test_015_write_and_read(mutable_database): assert new_rec.installed == rec.installed +def test_016_roundtrip_spliced_spec(mutable_database): + build_spec = spack.concretize.concretize_one("splice-t") + replacement = spack.concretize.concretize_one("splice-h+foo") + spec = build_spec.splice(replacement) + + spack.store.STORE.db.add(spec) + spack.store.STORE.db._state_is_inconsistent = True # force re-read + + _, spec_record = spack.store.STORE.db.query_by_spec_hash(spec.dag_hash()) + _, buildspec_record = spack.store.STORE.db.query_by_spec_hash(spec.build_spec.dag_hash()) + + assert spec_record.spec == spec + assert spec_record.spec.build_spec == spec.build_spec + assert buildspec_record # buildspec needs to be recorded in db + + def test_017_write_and_read_without_uuid(mutable_database, monkeypatch): monkeypatch.setattr(spack.database, "_use_uuid", False) # write and read DB @@ -993,7 +1012,7 @@ def test_mark_failed(mutable_database, monkeypatch, tmp_path: pathlib.Path, capf """Add coverage to mark_failed.""" def _raise_exc(lock): - raise lk.LockTimeoutError("write", "/mock-lock", 1.234, 10) + raise lk.LockTimeoutError(lk.LockType.WRITE, "/mock-lock", 1.234, 10) with fs.working_dir(str(tmp_path)): s = spack.concretize.concretize_one("pkg-a") diff --git a/lib/spack/spack/test/detection.py b/lib/spack/spack/test/detection.py index b501acb448fb7c..1218907d7dc656 100644 --- a/lib/spack/spack/test/detection.py +++ b/lib/spack/spack/test/detection.py @@ -4,10 +4,13 @@ import collections import pathlib +import pytest + import spack.config import spack.detection import spack.detection.common import spack.detection.path +import spack.repo import spack.spec @@ -52,3 +55,38 @@ def test_dedupe_paths(tmp_path: pathlib.Path): assert spack.detection.path.dedupe_paths([str(x), str(y), str(z)]) == [str(x), str(y)] assert spack.detection.path.dedupe_paths([str(z), str(y), str(x)]) == [str(x), str(y)] assert spack.detection.path.dedupe_paths([str(y), str(z), str(x)]) == [str(y), str(x)] + + +@pytest.mark.usefixtures("mock_packages") +def test_detect_specs_deduplicates_across_prefixes(tmp_path, monkeypatch): + """Tests that the same spec detected at two different prefixes should yield only one result. + + Returning both causes duplicate externals in packages.yaml and non-deterministic hashes + during concretization. + """ + # Create two independent bin/ directories, each containing the same executable name. + prefix_a = tmp_path / "prefix_a" + prefix_b = tmp_path / "prefix_b" + (prefix_a / "bin").mkdir(parents=True) + (prefix_b / "bin").mkdir(parents=True) + exe_a = prefix_a / "bin" / "cmake" + exe_b = prefix_b / "bin" / "cmake" + exe_a.touch() + exe_b.touch() + + cmake_cls = spack.repo.PATH.get_pkg_class("cmake") + + # Patch determine_spec_details to always return the same spec, regardless of prefix. + @classmethod + def _same_spec(cls, prefix, exes_in_prefix): + return spack.spec.Spec("cmake@3.17.1") + + monkeypatch.setattr(cmake_cls, "determine_spec_details", _same_spec) + + finder = spack.detection.path.ExecutablesFinder() + detected = finder.detect_specs( + pkg=cmake_cls, paths=[str(exe_a), str(exe_b)], repo_path=spack.repo.PATH + ) + + # Both prefixes produce cmake@3.17.1; only the first should be kept. + assert len(detected) == 1 diff --git a/lib/spack/spack/test/directives.py b/lib/spack/spack/test/directives.py index 8a129140ad95e6..d5c181c5c07d22 100644 --- a/lib/spack/spack/test/directives.py +++ b/lib/spack/spack/test/directives.py @@ -10,6 +10,9 @@ import spack.repo import spack.spec import spack.version +from spack.directives import _make_when_spec, depends_on, extends, patch +from spack.directives_meta import DirectiveDictDescriptor, DirectiveMeta +from spack.spec import Spec def test_false_directives_do_not_exist(mock_packages): @@ -212,3 +215,82 @@ def test_direct_dependencies_from_when_context_are_retained(mock_packages): assert spack.spec.Spec("%pkg-c") in pkg_cls.dependencies # Nested ^foo followed by ^foo %gcc assert spack.spec.Spec("^pkg-c %gcc") in pkg_cls.dependencies + + +def test_directives_meta_combine_when(): + x, y, z = "+x ^dep +a", "+y ^dep +b", "+z" + assert _make_when_spec((x, y, z)) == Spec("+x +y +z ^dep +a +b") + assert _make_when_spec((x, y)) == Spec("+x +y ^dep +a +b") + assert _make_when_spec((x,)) == Spec("+x ^dep +a") + + +def test_directive_descriptor_init(): + # when `pkg.variants` is initialized, only the `variant` directive should run + variants = DirectiveDictDescriptor("variants") + assert variants.directives_to_run == ["variant"] + assert variants.dicts_to_init == ["variants"] + + # when `pkg.dependencies` is initialized, `depends_on` and `extends` should run, and also + # `pkg.extendees` should be initialized + dependencies = DirectiveDictDescriptor("dependencies") + assert dependencies.directives_to_run == ["depends_on", "extends"] + assert dependencies.dicts_to_init == ["dependencies", "extendees"] + + # when `pkg.provided` is initialized, so should `pkg.provided_together`, and only the + # provides directive should run + provided = DirectiveDictDescriptor("provided") + assert provided.directives_to_run == ["provides"] + assert provided.dicts_to_init == ["provided", "provided_together"] + + # idem for `pkg.provided_together` + provided_together = DirectiveDictDescriptor("provided_together") + assert provided_together.directives_to_run == ["provides"] + assert provided_together.dicts_to_init == ["provided", "provided_together"] + + # when specifying patches on dependencies with `depends_on` and `extends`, the `pkg.patches` + # dict is not affects -- they are stored on a Dependency object. + patches = DirectiveDictDescriptor("patches") + assert patches.directives_to_run == ["patch"] + assert patches.dicts_to_init == ["patches"] + + +def test_directive_laziness(): + class ExamplePackage(metaclass=DirectiveMeta): + name = "example-package" + depends_on("foo") + extends("bar", when="+bar") + + # Initially, no directive dicts are initialized + assert ExamplePackage._dependencies is None # type: ignore + assert ExamplePackage._extendees is None # type: ignore + assert ExamplePackage._variants is None # type: ignore + + # Only when we access the dependencies descriptor, the relevant dicts (dependencies, extendees) + # are initialized, while others remain None + dependencies = ExamplePackage.dependencies # type: ignore + assert type(ExamplePackage._dependencies) is dict # type: ignore + assert type(ExamplePackage._extendees) is dict # type: ignore + assert ExamplePackage._variants is None # type: ignore + + # The dependencies dict is populated with the expected entries + assert "foo" in dependencies[spack.spec.Spec()] + assert "bar" in dependencies[spack.spec.Spec("+bar")] + + +def test_patched_dependencies_sets_class_attribute(): + sha256 = "a" * 64 + + class PatchesDependencies(metaclass=DirectiveMeta): + name = "patches-dependencies" + depends_on("dependency", patches=patch("https://example.com/diff.patch", sha256=sha256)) + + assert PatchesDependencies._patches_dependencies is True + assert not PatchesDependencies.patches # type: ignore + + class DoesNotPatchDependencies(metaclass=DirectiveMeta): + name = "does-not-patch-dependencies" + fullname = "does-not-patch-dependencies" + patch("https://example.com/diff.patch", sha256=sha256) + + assert DoesNotPatchDependencies._patches_dependencies is False + assert DoesNotPatchDependencies.patches # type: ignore diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index c74827f081dff1..d5f9b9a3bf207b 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -5,6 +5,7 @@ """ This test verifies that the Spack directory layout works properly. """ + import os import pathlib from pathlib import Path diff --git a/lib/spack/spack/test/env.py b/lib/spack/spack/test/env.py index 1aed072fc4fd5d..a25fffde003691 100644 --- a/lib/spack/spack/test/env.py +++ b/lib/spack/spack/test/env.py @@ -2,7 +2,9 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Test environment internals without CLI""" + import filecmp +import json import os import pathlib import pickle @@ -15,12 +17,16 @@ import spack.platforms import spack.solver.asp import spack.spec +import spack.spec_parser +from spack.enums import ConfigScopePriority +from spack.environment import SpackEnvironmentConfigError from spack.environment.environment import ( EnvironmentManifestFile, SpackEnvironmentViewError, _error_on_nonempty_view_dir, ) from spack.environment.list import UndefinedReferenceError +from spack.traverse import traverse_nodes pytestmark = [ pytest.mark.not_on_windows("Envs are not supported on windows"), @@ -59,10 +65,9 @@ def test_hash_change_no_rehash_concrete(tmp_path: pathlib.Path, config): env.concretize() # rewrite the hash - old_hash = env.concretized_order[0] - new_hash = "abc" + old_hash, new_hash = env.concretized_roots[0].hash, "abc" env.specs_by_hash[old_hash]._hash = new_hash # type: ignore[attr-defined] - env.concretized_order[0] = new_hash + env.concretized_roots[0].hash = new_hash env.specs_by_hash[new_hash] = env.specs_by_hash[old_hash] del env.specs_by_hash[old_hash] env.write() @@ -71,9 +76,10 @@ def test_hash_change_no_rehash_concrete(tmp_path: pathlib.Path, config): read_in = ev.Environment(env_path) # Ensure read hashes are used (rewritten hash seen on read) - assert read_in.concretized_order - assert read_in.concretized_order[0] in read_in.specs_by_hash - _hash = read_in.specs_by_hash[read_in.concretized_order[0]]._hash # type: ignore[attr-defined] + hashes = [x.hash for x in read_in.concretized_roots] + assert hashes + assert hashes[0] in read_in.specs_by_hash + _hash = read_in.specs_by_hash[hashes[0]]._hash # type: ignore[attr-defined] assert _hash == new_hash @@ -285,7 +291,12 @@ def test_update_default_view(init_view, update_value, tmp_path: pathlib.Path, co link_type: symlink """, "./another-view", - {"root": "./another-view", "select": ["%gcc"], "link_type": "symlink"}, + { + "root": "./another-view", + "select": ["%gcc"], + "link_type": "symlink", + "link_dirs": True, + }, ), ( """ @@ -299,7 +310,7 @@ def test_update_default_view(init_view, update_value, tmp_path: pathlib.Path, co link_type: symlink """, True, - {"root": "./view-gcc", "select": ["%gcc"], "link_type": "symlink"}, + {"root": "./view-gcc", "select": ["%gcc"], "link_type": "symlink", "link_dirs": True}, ), ], ) @@ -552,9 +563,8 @@ def test_environment_concretizer_scheme_used( ) mutable_config.set("concretizer:unify", unify_in_lower_scope) assert mutable_config.get("concretizer:unify") == unify_in_lower_scope - with ev.Environment(manifest.parent) as e: + with ev.Environment(manifest.parent): assert mutable_config.get("concretizer:unify") == unify_in_spack_yaml - assert e.unify == unify_in_spack_yaml @pytest.mark.parametrize("unify_in_config", [True, False, "when_possible"]) @@ -572,8 +582,8 @@ def test_environment_config_scheme_used(tmp_path: pathlib.Path, unify_in_config) ) with spack.config.override("concretizer:unify", unify_in_config): - with ev.Environment(manifest.parent) as e: - assert e.unify == unify_in_config + with ev.Environment(manifest.parent): + assert spack.config.CONFIG.get("concretizer:unify") == unify_in_config @pytest.mark.parametrize( @@ -795,14 +805,15 @@ def test_env_with_include_def_missing(mutable_mock_env_path): @pytest.mark.regression("41292") -def test_deconcretize_then_concretize_does_not_error(mutable_mock_env_path): +@pytest.mark.parametrize("unify", ["true", "false", "when_possible"]) +def test_deconcretize_then_concretize_does_not_error(mutable_mock_env_path, unify): """Tests that, after having deconcretized a spec, we can reconcretize an environment which has 2 or more user specs mapping to the same concrete spec. """ mutable_mock_env_path.mkdir() spack_yaml = mutable_mock_env_path / ev.manifest_name spack_yaml.write_text( - """spack: + f"""spack: specs: # These two specs concretize to the same hash - pkg-c @@ -810,15 +821,30 @@ def test_deconcretize_then_concretize_does_not_error(mutable_mock_env_path): # Spec used to trigger the bug - pkg-a concretizer: - unify: true + unify: {unify} """ ) e = ev.Environment(mutable_mock_env_path) + # Initial state + assert len(e.user_specs) == 3 + assert len(e.concretized_roots) == 0 + with e: e.concretize() - e.deconcretize(spack.spec.Spec("pkg-a"), concrete=False) + assert len(e.user_specs) == 3 + assert len(e.concretized_roots) == 3 + assert all(x.new for x in e.concretized_roots) + + e.deconcretize_by_user_spec(spack.spec.Spec("pkg-a")) + assert len(e.user_specs) == 3 + assert len(e.concretized_roots) == 2 + assert all(x.new for x in e.concretized_roots) + e.concretize() - assert len(e.concrete_roots()) == 3 + assert len(e.user_specs) == 3 + assert len(e.concretized_roots) == 3 + assert all(x.new for x in e.concretized_roots) + all_root_hashes = {x.dag_hash() for x in e.concrete_roots()} assert len(all_root_hashes) == 2 @@ -1179,7 +1205,10 @@ def test_toolchains_as_matrix_dimension(unify, tmp_path: pathlib.Path, mutable_c @pytest.mark.parametrize("unify", ["true", "false", "when_possible"]) -def test_using_toolchain_as_requirement(unify, tmp_path: pathlib.Path, mutable_config): +@pytest.mark.parametrize("requirement_type", ["require", "prefer"]) +def test_using_toolchain_as_requirement( + unify, requirement_type, tmp_path: pathlib.Path, mutable_config +): """Tests using a toolchain as a default requirement in an environment""" spack_yaml = f""" spack: @@ -1191,7 +1220,7 @@ def test_using_toolchain_as_requirement(unify, tmp_path: pathlib.Path, mutable_c {MIXED_TOOLCHAIN} packages: all: - require: + {requirement_type}: - "%mixed-toolchain" concretizer: unify: {unify} @@ -1583,7 +1612,8 @@ def test_ids_when_using_toolchain_twice_in_a_spec(tmp_path, mutable_config): manifest.write_text(spack_yaml) with ev.Environment(tmp_path): # We rely on this behavior when emitting facts for the solver - s = spack.spec.Spec("mpileaks %gnu ^callpath %gnu") + toolchains = spack.config.CONFIG.get("toolchains", {}) + s = spack.spec_parser.parse("mpileaks %gnu ^callpath %gnu", toolchains=toolchains)[0] assert id(s["gcc"]) != id(s["callpath"]["gcc"]) @@ -1630,3 +1660,481 @@ def test_installed_specs_disregards_deprecation(tmp_path, mutable_config): if node.satisfies("%c"): assert node.satisfies("%c=gcc@12"), node.tree() assert not node.satisfies("%c=gcc@7"), node.tree() + + +@pytest.fixture() +def create_temporary_manifest(tmp_path): + manifest_path = tmp_path / "spack.yaml" + + def _create(spack_yaml: str): + manifest_path.write_text(spack_yaml) + return EnvironmentManifestFile(tmp_path) + + return _create + + +@pytest.mark.usefixtures("mutable_config") +class TestEnvironmentGroups: + """Tests for the environment "groups" feature""" + + def test_manifest_and_groups(self, create_temporary_manifest): + """Tests a basic case of reading groups from a manifest file""" + manifest = create_temporary_manifest( + """ + spack: + specs: + - mpileaks + - group: compiler + matrix: + - [gcc@14] + - group: apps + needs: [compiler] + specs: + - matrix: + - [mpileaks] + - ["%gcc@14"] + - mpich + - libelf + """ + ) + # Check manifest properties + assert set(manifest.groups()) == {"default", "compiler", "apps"} + + assert manifest.user_specs(group="default") == manifest.user_specs() + assert manifest.user_specs() == ["mpileaks", "libelf"] + assert manifest.user_specs(group="compiler") == [{"matrix": [["gcc@14"]]}] + assert manifest.user_specs(group="apps") == [ + {"matrix": [["mpileaks"], ["%gcc@14"]]}, + "mpich", + ] + + assert manifest.needs(group="default") == () + assert manifest.needs(group="compiler") == () + assert manifest.needs(group="apps") == ("compiler",) + + # Check user specs within the environment + e = ev.Environment(manifest.manifest_dir) + assert e.user_specs.specs == [spack.spec.Spec("mpileaks"), spack.spec.Spec("libelf")] + + compiler_specs = e.user_specs_by(group="compiler") + assert compiler_specs.name == "specs:compiler" + assert compiler_specs.specs == [spack.spec.Spec("gcc@14")] + + apps_specs = e.user_specs_by(group="apps") + assert apps_specs.name == "specs:apps" + assert apps_specs.specs == [spack.spec.Spec("mpileaks %gcc@14"), spack.spec.Spec("mpich")] + + def test_cannot_define_group_twice(self, create_temporary_manifest): + """Tests that defining the same group twice raises an error""" + with pytest.raises(SpackEnvironmentConfigError, match="defined more than once"): + create_temporary_manifest( + """ + spack: + specs: + - group: compiler + matrix: + - [gcc@14] + - group: compiler + matrix: + - [llvm@20] +""" + ) + + def test_matrix_can_be_expanded_in_groups(self, create_temporary_manifest): + """Tests that definitions can be expanded also for matrix groups""" + manifest = create_temporary_manifest( + """ +spack: + definitions: + - compilers: ["%gcc", "%clang"] + - desired_specs: ["mpileaks@2.1"] + specs: + - group: apps + specs: + - matrix: + - [$desired_specs] + - [$compilers] + - mpich +""" + ) + e = ev.Environment(manifest.manifest_dir) + assert e.user_specs.specs == [] + assert e.user_specs_by(group="apps").specs == [ + spack.spec.Spec("mpileaks@2.1 %gcc"), + spack.spec.Spec("mpileaks@2.1 %clang"), + spack.spec.Spec("mpich"), + ] + + def test_environment_without_groups_use_lockfile_v6(self, create_temporary_manifest): + manifest = create_temporary_manifest( + """ +spack: + specs: + - mpileaks + - pkg-a +""" + ) + with ev.Environment(manifest.manifest_dir) as e: + e.concretize() + lockfile_data = e._to_lockfile_dict() + assert lockfile_data["_meta"]["lockfile-version"] == 6 + assert all("group" not in x for x in lockfile_data["roots"]) + + def test_independent_groups_concretization(self, create_temporary_manifest): + """Tests that groups of specs without dependencies among them can be concretized + correctly + """ + manifest = create_temporary_manifest( + """ + spack: + specs: + - mpileaks + - group: compiler + matrix: + - [gcc@14] + - libelf + """ + ) + + with ev.Environment(manifest.manifest_dir) as e: + e.concretize() + roots = e.concrete_roots() + assert len(roots) == 3 + + default_specs = list(e.concretized_specs_by(group="default")) + assert len(default_specs) == 2 + + compiler_specs = list(e.concretized_specs_by(group="compiler")) + assert len(compiler_specs) == 1 + + def test_independent_group_dont_reuse(self, create_temporary_manifest): + """Tests that there is no cross-groups reuse among groups of specs without dependencies.""" + manifest = create_temporary_manifest( + """ + spack: + specs: + - mpileaks@2.2 + - group: app + matrix: + - [mpileaks] + """ + ) + + with ev.Environment(manifest.manifest_dir) as e: + e.concretize() + + _, default_mpileaks = list(e.concretized_specs_by(group="default"))[0] + assert default_mpileaks.satisfies("@2.2") + + _, app_mpileaks = list(e.concretized_specs_by(group="app"))[0] + assert app_mpileaks.satisfies("@2.3") + + def test_relying_on_a_dependency_group(self, create_temporary_manifest): + """Tests that a group of specs that would not concretize without a dependency group + works correctly. + """ + manifest = create_temporary_manifest( + """ + spack: + specs: + - group: app + matrix: + - [mpileaks] + - ["%c,cxx=gcc@14"] + """ + ) + + # We have no gcc@14 configured, so this will raise an error + with ev.Environment(manifest.manifest_dir) as e: + with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): + e.concretize() + + manifest = create_temporary_manifest( + """ + spack: + specs: + - group: compiler + specs: + - gcc@14 + - group: mpileaks + needs: [compiler] + matrix: + - [mpileaks] + - ["%c,cxx=gcc@14"] + """ + ) + + # In this case gcc@14 is taken from the "needed" group + with ev.Environment(manifest.manifest_dir) as e: + e.concretize() + + _, gcc = next(iter(e.concretized_specs_by(group="compiler"))) + assert gcc.satisfies("gcc@14") + _, mpileaks = next(iter(e.concretized_specs_by(group="mpileaks"))) + assert mpileaks["c"].dag_hash() == gcc.dag_hash() + + def test_manifest_can_contain_config_override(self, mutable_config, create_temporary_manifest): + manifest = create_temporary_manifest( + """ + spack: + concretizer: + unify: False + specs: + - group: compiler + override: + concretizer: + unify: True + """ + ) + + with ev.Environment(manifest.manifest_dir) as e: + assert mutable_config.get_config("concretizer")["unify"] is False + + # Assert the internal scope works when used manually + override = manifest.config_override(group="compiler") + mutable_config.push_scope( + override, priority=ConfigScopePriority.ENVIRONMENT_SPEC_GROUPS + ) + assert mutable_config.get_config("concretizer")["unify"] is True + mutable_config.remove_scope(override.name) + assert mutable_config.get_config("concretizer")["unify"] is False + + # Assert the context manager works too + with e.config_override_for_group(group="compiler"): + assert mutable_config.get_config("concretizer")["unify"] is True + assert mutable_config.get_config("concretizer")["unify"] is False + + def test_overriding_concretization_properties_per_group(self, create_temporary_manifest): + manifest = create_temporary_manifest( + """ + spack: + concretizer: + unify: True + specs: + - group: compiler + specs: + - gcc@14 + - group: scalapacks + needs: [compiler] + matrix: + - [netlib-scalapack] + - ["%mpi=mpich", "%mpi=mpich2"] + - ["%lapack=openblas-with-lapack", "%lapack=netlib-lapack"] + override: + concretizer: + unify: False + packages: + c: + prefer: [gcc@14] + cxx: + prefer: [gcc@14] + fortran: + prefer: [gcc@14] + """ + ) + + with ev.Environment(manifest.manifest_dir) as e: + e.concretize() + + assert len(list(e.concretized_specs_by(group="compiler"))) == 1 + + gcc = next(x for _, x in e.concretized_specs_by(group="compiler")) + assert gcc.satisfies("gcc@14") and not gcc.external + assert gcc.satisfies("%c,cxx=gcc") + gcc_hash = gcc.dag_hash() + + assert len(list(e.concretized_specs_by(group="scalapacks"))) == 4 + scalapacks = [x for _, x in e.concretized_specs_by(group="scalapacks")] + for node in traverse_nodes(scalapacks, deptype=("link", "run")): + assert node.satisfies(f"%[when=c]c=gcc/{gcc_hash}") + assert node.satisfies(f"%[when=cxx]cxx=gcc/{gcc_hash}") + assert node.satisfies(f"%[when=fortran]fortran=gcc/{gcc_hash}") + + def test_missing_needs_group_gives_clear_error(self, create_temporary_manifest): + """Tests that referencing a non-existent group in 'needs' gives a clear error message + that includes the name of the blocked group and the missing dependency. + """ + manifest = create_temporary_manifest( + """ +spack: + specs: + - group: apps + needs: [nonexistent] + specs: + - mpileaks +""" + ) + with ev.Environment(manifest.manifest_dir) as e: + with pytest.raises( + ev.SpackEnvironmentConfigError, match=r"but 'nonexistent' is not a defined group" + ): + e.concretize() + + def test_cyclic_group_dependencies_give_clear_error(self, create_temporary_manifest): + """Tests that cyclic group dependencies give a clear error message that mentions + the groups involved in the cycle. + """ + manifest = create_temporary_manifest( + """ +spack: + specs: + - group: alpha + needs: [beta] + specs: + - mpileaks + - group: beta + needs: [alpha] + specs: + - zlib +""" + ) + with ev.Environment(manifest.manifest_dir) as e: + with pytest.raises(ev.SpackEnvironmentConfigError, match=r"among groups: alpha, beta"): + e.concretize() + + def test_from_lockfile_preserves_groups(self, tmp_path): + """Tests that EnvironmentManifestFile.from_lockfile reconstructs groups correctly + from a v7 lockfile that contains group information in its roots. + """ + lockfile_data = { + "_meta": {"file-type": "spack-lockfile", "lockfile-version": 7, "specfile-version": 5}, + "roots": [ + {"hash": "aaa", "spec": "mpileaks", "group": "default"}, + {"hash": "bbb", "spec": "libelf", "group": "default"}, + {"hash": "ccc", "spec": "gcc@14", "group": "compilers"}, + ], + "concrete_specs": {}, + } + lockfile_path = tmp_path / "spack.lock" + lockfile_path.write_text(json.dumps(lockfile_data)) + + manifest = EnvironmentManifestFile.from_lockfile(tmp_path) + + # The reconstructed manifest must have both groups + assert set(manifest.groups()) == {"default", "compilers"} + assert manifest.user_specs(group="default") == ["mpileaks", "libelf"] + assert manifest.user_specs(group="compilers") == ["gcc@14"] + + def test_from_lockfile_without_groups_stays_default(self, tmp_path): + """Tests that a lockfile without group info (v6 and earlier) reconstructs all specs + into the default group only. + """ + lockfile_data = { + "_meta": {"file-type": "spack-lockfile", "lockfile-version": 6, "specfile-version": 5}, + "roots": [{"hash": "aaa", "spec": "mpileaks"}, {"hash": "bbb", "spec": "libelf"}], + "concrete_specs": {}, + } + lockfile_path = tmp_path / "spack.lock" + lockfile_path.write_text(json.dumps(lockfile_data)) + + manifest = EnvironmentManifestFile.from_lockfile(tmp_path) + + assert set(manifest.groups()) == {"default"} + assert manifest.user_specs(group="default") == ["mpileaks", "libelf"] + + +@pytest.mark.regression("51995") +def test_mixed_compilers_and_libllvm(tmp_path, config): + """Tests that we divide virtual nodes correctly among unification sets. + + This test concretizes a unified environment where one package uses gcc as a C++ compiler + and depends on llvm as a provider of libllvm, while the other package uses llvm as a C++ + compiler. + """ + spack_yaml = """ +spack: + specs: + - paraview %cxx=llvm + - mesa %cxx=gcc %libllvm=llvm + packages: + c: + prefer: + - gcc + cxx: + prefer: + - gcc + gcc:: + externals: + - spec: gcc@13.2.0 languages:='c,c++,fortran' + prefix: /path + extra_attributes: + compilers: + c: /path/bin/gcc + cxx: /path/bin/g++ + fortran: /path/bin/gfortran + llvm:: + externals: + - spec: llvm@20.1.8+clang+flang+lld+lldb + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + concretizer: + unify: true +""" + manifest = tmp_path / "spack.yaml" + manifest.write_text(spack_yaml) + with ev.Environment(tmp_path) as e: + e.concretize() + + for x in e.concrete_roots(): + if x.name == "mesa": + mesa = x + else: + paraview = x + + assert paraview.satisfies("%cxx=llvm@20") + assert paraview.satisfies(f"%{mesa}") + assert mesa.satisfies("%cxx=gcc %libllvm=llvm") + assert paraview["cxx"].dag_hash() == mesa["libllvm"].dag_hash() + + +@pytest.mark.regression("51512") +def test_unified_environment_with_mixed_compilers_and_fortran(tmp_path, config): + """Tests that we can concretize a unified environment using two C/C++ compilers for the root + specs and GCC for Fortran, where both roots depend on Fortran. + """ + spack_yaml = """ + spack: + specs: + - mpich %c,cxx=llvm + - openblas %c,fortran=gcc + packages: + gcc:: + externals: + - spec: gcc@13.2.0 languages:='c,c++,fortran' + prefix: /path + extra_attributes: + compilers: + c: /path/bin/gcc + cxx: /path/bin/g++ + fortran: /path/bin/gfortran + llvm:: + externals: + - spec: llvm@20.1.8+clang~flang + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + concretizer: + unify: true + """ + manifest = tmp_path / "spack.yaml" + manifest.write_text(spack_yaml) + with ev.Environment(tmp_path) as e: + e.concretize() + + for x in e.concrete_roots(): + if x.name == "mpich": + mpich = x + else: + openblas = x + + assert mpich.satisfies("%c,cxx=llvm") + assert mpich.satisfies("%fortran=gcc") + assert openblas.satisfies("%c,fortran=gcc") + assert mpich["fortran"].dag_hash() == openblas["fortran"].dag_hash() diff --git a/lib/spack/spack/test/environment/mutate.py b/lib/spack/spack/test/environment/mutate.py index bc4677ab635b44..d9adf6a6273fdb 100644 --- a/lib/spack/spack/test/environment/mutate.py +++ b/lib/spack/spack/test/environment/mutate.py @@ -63,7 +63,7 @@ def test_mutate_internals(dep, orig_constraint, mutated_constraint): selector = spack.spec.Spec("cmake") mutator = spack.spec.Spec(mutated_constraint) - env.mutate(selector=selector, mutator=mutator) + env.mutate(selectors=[selector], mutators=[mutator]) cmake_spec.mutate(mutator) for spec in env.all_specs_generator(): @@ -81,6 +81,49 @@ def test_mutate_internals(dep, orig_constraint, mutated_constraint): assert root_spec.dag_hash() == new_hash +def test_mutate_internals_multiple_mutations(): + """ + Check that Environment.mutate correctly applies multiple mutations to different selected Specs. + """ + ev.create("test") + env = ev.read("test") + + root = "cmake-client+truthy os=debian6 %cmake@3.23.1 os=debian6" + env.add(root) + env.concretize() + + planned_mutations = [ + ("cmake", "@3.4.3"), + ("cmake-client", "~truthy"), + ("platform=test", "os=redhat6"), + ] + + orig_hash = next(env.roots()).dag_hash() + + selectors, mutators = zip( + *[(spack.spec.Spec(s), spack.spec.Spec(m)) for s, m in planned_mutations] + ) + + with pytest.raises(ValueError, match="Length mismatch: selectors"): + env.mutate(selectors=[], mutators=mutators) + + with pytest.raises(ValueError, match="Length mismatch: validators"): + env.mutate(selectors=selectors, mutators=mutators, validators=["cmake@3.4.3"]) + + with pytest.raises(ValueError, match="Length mismatch: msgs"): + env.mutate(selectors=selectors, mutators=mutators, msgs=["A message"]) + + env.mutate(selectors=selectors, mutators=mutators) + + for selector, mutated_constraint in planned_mutations: + for spec in env.all_specs_generator(): + if spec.satisfies(selector): + assert spec.satisfies(mutated_constraint) + + new_hash = next(env.roots()).dag_hash() + assert new_hash != orig_hash + + @pytest.mark.parametrize("constraint", ["foo", "foo.bar", "foo%cmake@1.0", "foo@1.1:", "foo/abc"]) def test_mutate_spec_invalid(constraint): spec = spack.concretize.concretize_one("cmake-client") diff --git a/lib/spack/spack/test/error_messages.py b/lib/spack/spack/test/error_messages.py index 795145303d6888..cde15404989028 100644 --- a/lib/spack/spack/test/error_messages.py +++ b/lib/spack/spack/test/error_messages.py @@ -10,6 +10,8 @@ import pytest +import spack.vendor.archspec.cpu + import spack.config import spack.error import spack.repo @@ -540,3 +542,89 @@ def test_errmsg_requirements_external_mismatch(concretize_scope, test_repo): with expect_failure_and_print(should_mention=important_points): concretize_one("t1") + + +@pytest.mark.parametrize("section", ["prefer", "require"]) +def test_warns_on_compiler_constraint_in_all(concretize_scope, mock_packages, section): + """Compiler constraints under packages:all: are a footgun and should warn.""" + update_packages_config(f"packages:\n all:\n {section}:\n - '%c=gcc'\n") + with pytest.warns(UserWarning, match="packages: all:"): + concretize_one("gmake") + + +@pytest.mark.regression("52209") +def test_unknown_concrete_target_in_input_spec(concretize_scope, test_repo): + """Tests that an input spec with an unknown concrete target raises a clear error naming + the bad target, rather than a confusing 'cannot satisfy constraint' solver error. + """ + spec_str = "x4 target=not-a-real-uarch" + with pytest.raises(spack.error.SpackError) as exc_info: + concretize_one(spec_str) + check_error(str(exc_info.value), should_mention=[spec_str, "not a known target"]) + + +@pytest.mark.regression("52209") +def test_require_single_unknown_target_errors(concretize_scope, test_repo): + """Tests that a single-option require with an unknown target raises a clear error.""" + target_str = "target=not-a-real-uarch" + update_packages_config( + f"""\ +packages: + x4: + require: {target_str} +""" + ) + with pytest.raises(spack.error.SpackError) as exc_info: + concretize_one("x4") + check_error(str(exc_info.value), should_mention=[target_str, "unknown target"]) + + +@pytest.mark.regression("52209") +def test_require_all_unknown_targets_errors(concretize_scope, test_repo): + """Tests that a group where every option has an unknown target also raises a clear error.""" + update_packages_config( + """\ +packages: + x4: + require: + - any_of: ["target=not-a-real-uarch", "target=also-fake"] +""" + ) + with pytest.raises(spack.error.SpackError) as exc_info: + concretize_one("x4") + check_error( + str(exc_info.value), + should_mention=["target=not-a-real-uarch", "target=also-fake", "unknown target"], + ) + + +@pytest.mark.regression("52209") +@pytest.mark.skipif( + str(spack.vendor.archspec.cpu.host().family) != "x86_64", reason="test assumes x86_64 uarchs" +) +def test_require_mixed_unknown_and_valid_target_warns(concretize_scope, test_repo): + """Tests that a "require" group with at least one valid option just warns.""" + update_packages_config( + """\ +packages: + x4: + require: + - one_of: ["target=not-a-real-uarch", "target=x86_64"] +""" + ) + with pytest.warns(UserWarning, match="not-a-real-uarch"): + concretize_one("x4") + + +@pytest.mark.regression("52209") +def test_prefer_unknown_target_warns(concretize_scope, test_repo): + """A preference with an unknown target has the @: fallback, so it only warns.""" + update_packages_config( + """\ +packages: + x4: + prefer: ["target=not-a-real-uarch"] +""" + ) + with pytest.warns(UserWarning, match="not-a-real-uarch"): + concretize_one("x4") diff --git a/lib/spack/spack/test/externals.py b/lib/spack/spack/test/externals.py index a57a70297405b4..a9fbe1f10926ba 100644 --- a/lib/spack/spack/test/externals.py +++ b/lib/spack/spack/test/externals.py @@ -340,3 +340,35 @@ def test_external_node_completion( # Assert all nodes have the namespace set for node in spack.traverse.traverse_nodes(parser.all_specs()): assert node.namespace is not None + + +@pytest.mark.regression("52179") +def test_external_spec_single_valued_variant_type_is_corrected(): + """Tests that an external spec string including a single-valued variant is parsed correctly.""" + externals_dict = [ + {"spec": "dual-cmake-autotools@1.0 build_system=autotools", "prefix": "/usr/dual"} + ] + parser = ExternalSpecsParser(externals_dict, complete_node=complete_variants_and_architecture) + specs = parser.all_specs() + assert len(specs) == 1 + spec = specs[0] + + # Single-valued variants return the value, not a tuple of values + build_system_value = spec.variants["build_system"].value + assert build_system_value == "autotools", ( + f"Expected 'autotools' but got {build_system_value!r} " + f"(type: {type(build_system_value).__name__})" + ) + + +@pytest.mark.regression("52179") +def test_external_spec_multi_valued_variant_is_not_changed(): + """Tests that multi-valued variants in external specs are preserved as they are, even if the + definition in package.py says otherwise. + """ + # Package.py prescribes a single-valued variant in this case + externals_dict = [{"spec": "variant-values@1.0 v=foo,bar", "prefix": "/usr/variant-values"}] + parser = ExternalSpecsParser(externals_dict, complete_node=complete_variants_and_architecture) + specs = parser.all_specs() + assert len(specs) == 1 + assert specs[0].variants["v"].value == ("bar", "foo") diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 717bb0a1ead6e8..0eccf2dab412f4 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -27,6 +27,7 @@ _mock_transport_error = "Mock HTTP transport error" min_opt_string = ".".join(map(str, spack.util.git.MIN_OPT_VERSION)) +min_direct_commit = ".".join(map(str, spack.util.git.MIN_DIRECT_COMMIT_FETCH)) @pytest.fixture(params=[None, "1.8.5.2", "1.8.5.1", "1.7.10", "1.7.1", "1.7.0"]) @@ -111,6 +112,9 @@ def test_fetch( s = default_mock_concretization("git-test") monkeypatch.setitem(s.package.versions, Version("git"), t.args) + if type_of_test == "commit": + s.variants["commit"] = SingleValuedVariant("commit", t.args["commit"]) + # Enter the stage directory and check some properties with s.package.stage: with spack.config.override("config:verify_ssl", secure): @@ -241,8 +245,10 @@ def test_needs_stage(git): @pytest.mark.parametrize("get_full_repo", [True, False]) +@pytest.mark.parametrize("use_commit", [True, False]) def test_get_full_repo( get_full_repo, + use_commit, git_version, mock_git_repository, default_mock_concretization, @@ -254,16 +260,31 @@ def test_get_full_repo( if git_version < Version(min_opt_string): pytest.skip("Not testing get_full_repo for older git {0}".format(git_version)) + # newer git allows for direct commit fetching + can_use_direct_commit = git_version >= Version(min_direct_commit) + secure = True type_of_test = "tag-branch" t = mock_git_repository.checks[type_of_test] - s = default_mock_concretization("git-test") + spec_string = "git-test" + + s = default_mock_concretization(spec_string) + args = copy.copy(t.args) args["get_full_repo"] = get_full_repo monkeypatch.setitem(s.package.versions, Version("git"), args) + if use_commit: + git_exe = mock_git_repository.git_exe + url = mock_git_repository.url + commit = git_exe("ls-remote", url, t.revision, output=str).strip().split()[0] + s.variants["commit"] = SingleValuedVariant("commit", commit) + if can_use_direct_commit: + path = mock_git_repository.path + git_exe("-C", path, "config", "uploadpack.allowReachableSHA1InWant", "true") + with s.package.stage: with spack.config.override("config:verify_ssl", secure): s.package.do_stage() @@ -280,11 +301,27 @@ def test_get_full_repo( ncommits = len(commits) if get_full_repo: - assert nbranches >= 5 - assert ncommits == 2 + # default branch commit, plus checkout commit + assert ncommits == 2, commits + assert nbranches >= 5, branches else: - assert nbranches == 2 - assert ncommits == 1 + assert ncommits == 1, commits + if can_use_direct_commit: + if use_commit: + # only commit (detached state) + assert nbranches == 1, branches + else: + # tag, commit (detached state) + assert nbranches == 2, branches + else: + if use_commit: + # default branch, tag, commit (detached state) + # git does not have a rewind, avoid messing with git history by + # accepting detachment + assert nbranches == 3, branches + else: + # default branch plus tag + assert nbranches == 2, branches @pytest.mark.disable_clean_stage_check diff --git a/lib/spack/spack/test/installer.py b/lib/spack/spack/test/installer.py index 4a5a1955ca5f9a..daf952ae2778e6 100644 --- a/lib/spack/spack/test/installer.py +++ b/lib/spack/spack/test/installer.py @@ -19,6 +19,7 @@ import spack.error import spack.hooks import spack.installer as inst +import spack.installer_dispatch import spack.llnl.util.filesystem as fs import spack.llnl.util.lock as ulk import spack.llnl.util.tty as tty @@ -1082,10 +1083,10 @@ def test_install_fail_fast_on_detect(install_mockery, monkeypatch, capfd): installer.install() assert b_id in installer.failed, "Expected b to be marked as failed" - assert c_id in installer.failed, "Exepected c to be marked as failed" - assert ( - a_id not in installer.installed - ), "Package a cannot install due to its dependencies failing" + assert c_id in installer.failed, "Expected c to be marked as failed" + assert a_id not in installer.installed, ( + "Package a cannot install due to its dependencies failing" + ) # check that b's active process got killed when c failed assert f"{b_id} failed to install" in capfd.readouterr().err @@ -1389,3 +1390,24 @@ def test_print_install_test_log_failures( inst.print_install_test_log(pkg) out = capfd.readouterr()[0] assert "See test results at" in out + + +def test_fallback_to_old_installer_for_splicing(monkeypatch, mock_packages, mutable_config): + """Test that the old installer is used for spliced specs (unsupported in the new installer)""" + mutable_config.set("config:installer", "new") + spec = spack.concretize.concretize_one("splice-t") + dep = spack.concretize.concretize_one("splice-h+foo") + out = spec.splice(dep) + assert isinstance( + spack.installer_dispatch.create_installer([out.package]), inst.PackageInstaller + ) + + +@pytest.mark.disable_clean_stage_check +def test_log_files_preserved_on_error(install_mockery, mock_fetch, installer_variant): + """Test that the log file is preserved when an install error occurs.""" + pkg = spack.concretize.concretize_one("build-error").package + installer = spack.installer_dispatch.create_installer([pkg]) + with pytest.raises(spack.error.InstallError): + installer.install() + assert os.path.exists(pkg.log_path) diff --git a/lib/spack/spack/test/installer_build_graph.py b/lib/spack/spack/test/installer_build_graph.py index 0aa5ca30069920..c8c50ccf81c50a 100644 --- a/lib/spack/spack/test/installer_build_graph.py +++ b/lib/spack/spack/test/installer_build_graph.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for BuildGraph class in new_installer""" + import sys from typing import Dict, List, Tuple, Union @@ -584,3 +585,276 @@ def test_pruning_root_node_with_install_package_false( # dep1 should not appear in any mappings assert dep1_hash not in graph.parent_to_child assert dep1_hash not in graph.child_to_parent + + +@pytest.fixture +def specs_with_test_deps(): + """Create specs with test-typed dependencies. + + DAG structure: + root -> dep (link) + test_dep (test) + dep -> dep_test_dep (test) + """ + return create_dag( + nodes=["root", "dep", "test_dep", "dep_test_dep"], + edges=[ + ("root", "dep", ("build", "link")), + ("root", "test_dep", "test"), + ("dep", "dep_test_dep", "test"), + ], + ) + + +class TestBuildGraphTestDeps: + """Tests for BuildGraph handling of TEST-typed dependencies.""" + + def test_tests_false_excludes_test_deps( + self, specs_with_test_deps: Dict[str, Spec], temporary_store: Store + ): + """Test that tests=False excludes TEST-typed dependencies.""" + graph = BuildGraph( + specs=[specs_with_test_deps["root"]], + root_policy="auto", + dependencies_policy="auto", + include_build_deps=True, + install_package=True, + install_deps=True, + database=temporary_store.db, + tests=False, + ) + + assert specs_with_test_deps["dep"].dag_hash() in graph.nodes + assert specs_with_test_deps["test_dep"].dag_hash() not in graph.nodes + assert specs_with_test_deps["dep_test_dep"].dag_hash() not in graph.nodes + + def test_tests_root_includes_test_deps_for_root( + self, specs_with_test_deps: Dict[str, Spec], temporary_store: Store + ): + """Test that tests=[root_name] includes test deps only for the root package.""" + graph = BuildGraph( + specs=[specs_with_test_deps["root"]], + root_policy="auto", + dependencies_policy="auto", + include_build_deps=True, + install_package=True, + install_deps=True, + database=temporary_store.db, + tests=["root"], + ) + + assert specs_with_test_deps["dep"].dag_hash() in graph.nodes + assert specs_with_test_deps["test_dep"].dag_hash() in graph.nodes + # dep's test dep is NOT included because tests=["root"] only applies to "root" + assert specs_with_test_deps["dep_test_dep"].dag_hash() not in graph.nodes + + def test_tests_all_includes_test_deps_for_all( + self, specs_with_test_deps: Dict[str, Spec], temporary_store: Store + ): + """Test that tests=True includes TEST-typed deps for all packages.""" + graph = BuildGraph( + specs=[specs_with_test_deps["root"]], + root_policy="auto", + dependencies_policy="auto", + include_build_deps=True, + install_package=True, + install_deps=True, + database=temporary_store.db, + tests=True, + ) + + assert specs_with_test_deps["dep"].dag_hash() in graph.nodes + assert specs_with_test_deps["test_dep"].dag_hash() in graph.nodes + assert specs_with_test_deps["dep_test_dep"].dag_hash() in graph.nodes + + def test_mark_explicit_spec_excludes_build_only_deps( + self, specs_with_build_deps: Dict[str, Spec], temporary_store: Store + ): + """An installed-implicit spec in explicit_set should only traverse link/run deps, + not build-only deps.""" + root = specs_with_build_deps["root"] + install_spec_in_db(root, temporary_store) + assert temporary_store.db._data[root.dag_hash()].explicit is False + graph = BuildGraph( + specs=[root], + root_policy="auto", + dependencies_policy="auto", + include_build_deps=True, + install_package=True, + install_deps=True, + database=temporary_store.db, + explicit_set={root.dag_hash()}, + ) + # root should be in graph (not pruned) because it needs to be marked explicit. + assert root.dag_hash() in graph.nodes + # build-only dep should NOT be pulled in since root is already installed. + assert specs_with_build_deps["build_dep"].dag_hash() not in graph.nodes + + +class TestExpandBuildDeps: + """Tests for BuildGraph.expand_build_deps after a binary cache miss.""" + + def _make_graph(self, specs, root, temporary_store): + """Helper to create a BuildGraph with include_build_deps=False (auto policy).""" + return BuildGraph( + specs=[specs[root]], + root_policy="auto", + dependencies_policy="auto", + include_build_deps=False, + install_package=True, + install_deps=True, + database=temporary_store.db, + ) + + def _expand(self, graph, dag_hash, pending, db, tests=False): + """Call expand_build_deps under the DB read lock (as the real caller would).""" + with db.read_transaction(): + return graph.expand_build_deps([dag_hash], pending, db, tests) + + def test_expand_build_deps_adds_missing_deps(self, temporary_store: Store): + """A --build--> C --link--> D, A --link--> B. + Initial graph (auto, no build deps): A, B. + After expand: C, D added. D is leaf -> in pending_builds.""" + specs = create_dag( + nodes=["a", "b", "c", "d"], + edges=[("a", "b", "link"), ("a", "c", "build"), ("c", "d", "link")], + ) + graph = self._make_graph(specs, "a", temporary_store) + assert specs["c"].dag_hash() not in graph.nodes + assert specs["d"].dag_hash() not in graph.nodes + + pending: List[str] = [] + newly_added = self._expand(graph, specs["a"].dag_hash(), pending, temporary_store.db) + + assert specs["c"].dag_hash() in newly_added + assert specs["d"].dag_hash() in newly_added + assert specs["c"].dag_hash() in graph.nodes + assert specs["d"].dag_hash() in graph.nodes + # D is a leaf (no children), so it should be enqueued + assert specs["d"].dag_hash() in pending + # C waits on D, so it should NOT be enqueued + assert specs["c"].dag_hash() not in pending + + def test_expand_build_deps_shared_dep_already_in_graph(self, temporary_store: Store): + """A --link--> B, A --build--> C --link--> B. + Initial graph: A, B. After expand: C added with edge C->B.""" + specs = create_dag( + nodes=["a", "b", "c"], + edges=[("a", "b", "link"), ("a", "c", "build"), ("c", "b", "link")], + ) + graph = self._make_graph(specs, "a", temporary_store) + assert specs["b"].dag_hash() in graph.nodes + assert specs["c"].dag_hash() not in graph.nodes + + pending: List[str] = [] + newly_added = self._expand(graph, specs["a"].dag_hash(), pending, temporary_store.db) + + assert specs["c"].dag_hash() in newly_added + # C depends on B, so C->B edge should exist + assert specs["b"].dag_hash() in graph.parent_to_child[specs["c"].dag_hash()] + # B should list C as a parent + assert specs["c"].dag_hash() in graph.child_to_parent[specs["b"].dag_hash()] + # C waits on B, so not in pending + assert specs["c"].dag_hash() not in pending + + def test_expand_build_deps_skips_installed_in_db(self, temporary_store: Store): + """A --build--> C --link--> D. D installed in DB. + After expand: C added, D NOT added. No edge C->D. C in pending.""" + specs = create_dag(nodes=["a", "c", "d"], edges=[("a", "c", "build"), ("c", "d", "link")]) + install_spec_in_db(specs["d"], temporary_store) + graph = self._make_graph(specs, "a", temporary_store) + + pending: List[str] = [] + newly_added = self._expand(graph, specs["a"].dag_hash(), pending, temporary_store.db) + + assert specs["c"].dag_hash() in newly_added + assert specs["d"].dag_hash() not in newly_added + assert specs["d"].dag_hash() not in graph.nodes + # No edge from C to D (installed dep) + assert specs["d"].dag_hash() not in graph.parent_to_child.get(specs["c"].dag_hash(), set()) + # C has no uninstalled children, so it should be enqueued + assert specs["c"].dag_hash() in pending + + def test_expand_build_deps_skips_installed_in_session(self, temporary_store: Store): + """Same as above, but D in graph.done instead of DB.""" + specs = create_dag(nodes=["a", "c", "d"], edges=[("a", "c", "build"), ("c", "d", "link")]) + graph = self._make_graph(specs, "a", temporary_store) + graph.done.add(specs["d"].dag_hash()) + + pending: List[str] = [] + newly_added = self._expand(graph, specs["a"].dag_hash(), pending, temporary_store.db) + + assert specs["c"].dag_hash() in newly_added + assert specs["d"].dag_hash() not in newly_added + assert specs["d"].dag_hash() not in graph.nodes + assert specs["c"].dag_hash() in pending + + def test_expand_build_deps_reenqueues_original_when_all_deps_installed( + self, temporary_store: Store + ): + """A --build--> C. C installed in DB. + After expand: C NOT added. A re-enqueued (no uninstalled children).""" + specs = create_dag(nodes=["a", "c"], edges=[("a", "c", "build")]) + install_spec_in_db(specs["c"], temporary_store) + graph = self._make_graph(specs, "a", temporary_store) + + pending: List[str] = [] + newly_added = self._expand(graph, specs["a"].dag_hash(), pending, temporary_store.db) + + assert len(newly_added) == 0 + assert specs["a"].dag_hash() in pending + + def test_expand_build_deps_no_deadlock_on_installed_dep(self, temporary_store: Store): + """A --build--> C --link--> D. D installed in DB. + No edge C->D in parent_to_child. C in pending.""" + specs = create_dag(nodes=["a", "c", "d"], edges=[("a", "c", "build"), ("c", "d", "link")]) + install_spec_in_db(specs["d"], temporary_store) + graph = self._make_graph(specs, "a", temporary_store) + + pending: List[str] = [] + self._expand(graph, specs["a"].dag_hash(), pending, temporary_store.db) + + # No edge from C to D: installed deps get no edge, otherwise C is never scheduled + assert specs["d"].dag_hash() not in graph.parent_to_child.get(specs["c"].dag_hash(), set()) + assert specs["c"].dag_hash() in pending + + def test_has_unexpanded_build_deps_true(self, temporary_store: Store): + """A --build--> C, A --link--> B. With include_build_deps=False, C is not in the graph, + so has_unexpanded_build_deps returns True.""" + specs = create_dag(nodes=["a", "b", "c"], edges=[("a", "b", "link"), ("a", "c", "build")]) + graph = self._make_graph(specs, "a", temporary_store) + assert graph.has_unexpanded_build_deps(specs["a"].dag_hash()) + + def test_has_unexpanded_build_deps_false_shared(self, temporary_store: Store): + """A --(build,link)--> B. B is already in graph as link dep, + so has_unexpanded_build_deps returns False.""" + specs = create_dag(nodes=["a", "b"], edges=[("a", "b", ("build", "link"))]) + graph = self._make_graph(specs, "a", temporary_store) + assert not graph.has_unexpanded_build_deps(specs["a"].dag_hash()) + + def test_expand_build_deps_does_not_mark_in_graph_spec_as_done(self, temporary_store: Store): + """A --link--> B, A --link--> C, B --build--> C. + C is in the graph (link dep of A) and installed in DB (simulating an overwrite build + in progress). Expanding B's build deps should add edge B->C and NOT mark C as done.""" + specs = create_dag( + nodes=["a", "b", "c"], + edges=[("a", "b", "link"), ("a", "c", "link"), ("b", "c", "build")], + ) + graph = self._make_graph(specs, "a", temporary_store) + # C should be in graph as a link dep of A + assert specs["c"].dag_hash() in graph.nodes + + # Simulate overwrite: install C in DB after graph creation + install_spec_in_db(specs["c"], temporary_store) + + pending: List[str] = [] + self._expand(graph, specs["b"].dag_hash(), pending, temporary_store.db) + + c_hash = specs["c"].dag_hash() + b_hash = specs["b"].dag_hash() + # C must NOT be marked as done (it's still being overwrite-built) + assert c_hash not in graph.done + # Edge B->C must exist + assert c_hash in graph.parent_to_child[b_hash] + assert b_hash in graph.child_to_parent[c_hash] + # B should NOT be in pending (it still waits on C) + assert b_hash not in pending diff --git a/lib/spack/spack/test/installer_tui.py b/lib/spack/spack/test/installer_tui.py index 55f39eb3ddac3d..d398cd91706024 100644 --- a/lib/spack/spack/test/installer_tui.py +++ b/lib/spack/spack/test/installer_tui.py @@ -3,18 +3,21 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for the BuildStatus terminal UI in new_installer.py""" -import io -import os import sys -from typing import List, Optional, Tuple import pytest if sys.platform == "win32": pytest.skip("No Windows support", allow_module_level=True) + +import io +import os +from multiprocessing import Pipe +from typing import List, Optional, Tuple + import spack.new_installer as inst -from spack.new_installer import BuildStatus +from spack.new_installer import BuildStatus, StdinReader class MockConnection: @@ -64,7 +67,13 @@ def clear(self): def create_build_status( - is_tty: bool = True, terminal_cols: int = 80, terminal_rows: int = 24, total: int = 0 + is_tty: bool = True, + terminal_cols: int = 80, + terminal_rows: int = 24, + total: int = 0, + verbose: bool = False, + filter_padding: bool = False, + color: Optional[bool] = None, ) -> Tuple[BuildStatus, List[float], SimpleTextIOWrapper]: """Helper function to create BuildStatus with mocked dependencies""" fake_stdout = SimpleTextIOWrapper(tty=is_tty) @@ -83,6 +92,9 @@ def mock_get_terminal_size(): get_terminal_size=mock_get_terminal_size, get_time=mock_get_time, is_tty=is_tty, + verbose=verbose, + filter_padding=filter_padding, + color=color, ) return status, time_values, fake_stdout @@ -99,6 +111,27 @@ def add_mock_builds(status: BuildStatus, count: int) -> List[MockSpec]: class TestBasicStateManagement: """Test basic state management operations""" + def test_on_resize(self): + """Test that on_resize sets terminal_size_changed and update() fetches lazily""" + sizes = [os.terminal_size((80, 24))] + fake_stdout = SimpleTextIOWrapper(tty=True) + status = BuildStatus( + total=0, stdout=fake_stdout, get_terminal_size=lambda: sizes[-1], is_tty=True + ) + # terminal_size_changed is True from __init__; terminal_size is placeholder + assert status.terminal_size_changed is True + + # After on_resize the flag stays set and dirty is True + sizes.append(os.terminal_size((120, 40))) + status.on_resize() + assert status.terminal_size_changed is True + assert status.dirty is True + + # The actual size is fetched lazily on the first update() + status.update() + assert status.terminal_size == os.terminal_size((120, 40)) + assert status.terminal_size_changed is False + def test_add_build(self): """Test that add_build adds builds correctly""" status, _, _ = create_build_status(total=2) @@ -146,6 +179,64 @@ def test_update_state_failed(self): assert status.completed == 1 assert status.builds[build_id].finished_time == fake_time[0] + inst.CLEANUP_TIMEOUT + def test_remove_build(self): + """Test that remove_build removes the build from the display.""" + status, _, _ = create_build_status(total=2) + specs = add_mock_builds(status, 2) + build_id = specs[0].dag_hash() + + status.dirty = False + status.remove_build(build_id) + assert build_id not in status.builds + assert len(status.builds) == 1 + assert status.dirty is True + + def test_remove_build_resets_tracked(self): + """Test that removing the tracked build resets tracking to overview mode.""" + status, _, _ = create_build_status(total=1) + (spec,) = add_mock_builds(status, 1) + build_id = spec.dag_hash() + + status.tracked_build_id = build_id + status.overview_mode = False + status.remove_build(build_id) + assert status.tracked_build_id == "" + assert status.overview_mode is True + + def test_parse_log_summary(self, tmp_path): + """Test that parse_log_summary parses the build log and stores the summary.""" + status, _, _ = create_build_status() + (spec,) = add_mock_builds(status, 1) + build_id = spec.dag_hash() + + # Create a fake log file with an error + log_file = tmp_path / "build.log" + log_file.write_text("error: something went wrong\n") + + status.builds[build_id].log_path = str(log_file) + status.parse_log_summary(build_id) + assert status.builds[build_id].log_summary is not None + assert "error" in status.builds[build_id].log_summary.lower() + + def test_parse_log_summary_no_log_path(self): + """Test that parse_log_summary is a no-op when log_path is not set.""" + status, _, _ = create_build_status() + (spec,) = add_mock_builds(status, 1) + build_id = spec.dag_hash() + + status.parse_log_summary(build_id) + assert status.builds[build_id].log_summary is None + + def test_parse_log_summary_missing_file(self, tmp_path): + """Test that parse_log_summary is a no-op when log file doesn't exist.""" + status, _, _ = create_build_status() + (spec,) = add_mock_builds(status, 1) + build_id = spec.dag_hash() + + status.builds[build_id].log_path = str(tmp_path / "nonexistent.log") + status.parse_log_summary(build_id) + assert status.builds[build_id].log_summary is None + def test_update_progress(self): """Test that update_progress updates percentages""" status, _, _ = create_build_status() @@ -198,9 +289,10 @@ def test_non_tty_output(self): status.update_state(build_id, "finished") output = fake_stdout.getvalue() + assert "[+]" in output assert "mypackage" in output assert "1.0" in output - assert "finished" in output + assert "/fake/prefix/mypackage" in output # prefix is shown for finished builds # Non-TTY output should not contain ANSI escape codes assert "\033[" not in output @@ -267,9 +359,9 @@ def test_cursor_movement_vs_newlines(self): status.update() output1 = fake_stdout.getvalue() - # Count newlines (\n) and cursor movements (\033[1E = move down 1 line) + # Count newlines (\n) and cursor movements (\033[1B\r = move down 1 line) newlines1 = output1.count("\n") - cursor_moves1 = output1.count("\033[1E") + cursor_moves1 = output1.count("\033[1B\r") # Initially all lines should be newlines (nothing in history yet) assert newlines1 > 0 @@ -291,7 +383,7 @@ def test_cursor_movement_vs_newlines(self): output2 = fake_stdout.getvalue() newlines2 = output2.count("\n") - cursor_moves2 = output2.count("\033[1E") + cursor_moves2 = output2.count("\033[1B\r") # Should have newlines for the 2 finished builds persisted to history # and cursor movements for the active area (header + 3 active builds) @@ -723,22 +815,41 @@ def test_print_logs_discarded_when_not_tracked(self): # Nothing should be printed since we're tracking pkg1, not pkg2 assert fake_stdout.getvalue() == "" - def test_cannot_follow_failed_build(self): - """Test that navigation skips failed builds""" - status, _, _ = create_build_status(total=3) + def test_can_navigate_to_failed_build(self): + """Test that navigating to a failed build shows log summary and path""" + status, _, fake_stdout = create_build_status(total=3) specs = add_mock_builds(status, 3) - # Mark the middle build as failed + # Mark the middle build as failed and set log info status.update_state(specs[1].dag_hash(), "failed") + build_info = status.builds[specs[1].dag_hash()] + build_info.log_summary = "Error: something went wrong\n" + build_info.log_path = "/tmp/spack/pkg1.log" - # The failed build should have finished_time set - assert status.builds[specs[1].dag_hash()].finished_time is not None + # Navigate from pkg0 to next -- should land on failed pkg1 + status.tracked_build_id = specs[0].dag_hash() + next_id = status._get_next(1) + assert next_id == specs[1].dag_hash() + + # Actually navigate to it + status.next(1) + output = fake_stdout.getvalue() + assert "Log summary of pkg1" in output + assert "Error: something went wrong" in output + assert "/tmp/spack/pkg1.log" in output + + def test_navigation_skips_finished_build(self): + """Test that navigation skips successfully finished builds""" + status, _, _ = create_build_status(total=3) + specs = add_mock_builds(status, 3) + + # Mark the middle build as finished (successful) + status.update_state(specs[1].dag_hash(), "finished") - # Try to get next build, should skip the failed one + # Try to get next build, should skip the finished one status.tracked_build_id = specs[0].dag_hash() next_id = status._get_next(1) - # Should skip pkg1 (failed) and return pkg2 assert next_id == specs[2].dag_hash() @@ -913,6 +1024,68 @@ def test_update_state_finished_triggers_toggle_when_tracking(self): assert status.overview_mode is True assert status.tracked_build_id == "" + def test_partial_line_newline_on_toggle_and_next(self): + """Ensure newline is inserted before mode transitions when log doesn't end with newline.""" + status, _, fake_stdout = create_build_status(total=2) + specs = add_mock_builds(status, 2) + build_a, build_b = specs[0].dag_hash(), specs[1].dag_hash() + + # Follow a build, toggle back and forth between logs and overview mode, and receive logs + # that may or may not end with newlines. + status.next() + status.print_logs(build_a, b"checking for foo...") + status.toggle() + status.next() + status.print_logs(build_a, b"checking for bar... yes\n") + status.next(1) + status.print_logs(build_b, b"checking for baz...") + status.next(-1) + + written = fake_stdout.getvalue() + + # There shouldn't be any double newlines: + assert "\n\n" not in written + + # All partial and newline-terminated logs should be present with appropriate newlines: + assert "checking for foo...\n" in written + assert "checking for bar... yes\n" in written + assert "checking for baz...\n" in written + + @pytest.mark.parametrize("filter_padding", [True, False]) + def test_print_logs_filters_padding(self, filter_padding): + """print_logs strips path-padding placeholders before writing to stdout.""" + status, _, fake_stdout = create_build_status(filter_padding=filter_padding) + (spec,) = add_mock_builds(status, 1) + build_id = spec.dag_hash() + log_output = b"--with-foo=/base/__spack_path_placeholder__/__spack_path_placeholder__/bin" + + # track the build and print logs with the relevant path. + status.overview_mode = False + status.tracked_build_id = build_id + status.print_logs(build_id, log_output) + written = fake_stdout._buffer.getvalue() + + if filter_padding: + assert written == b"--with-foo=/base/[padded-to-59-chars]/bin" + else: + assert written == log_output + + @pytest.mark.parametrize("filter_padding", [True, False]) + def test_prefix_padding_filter_in_status(self, filter_padding): + """Test that prefix in status indicator applies padding filter.""" + padded_prefix = "/base/__spack_path_placeholder__/__spack_path_placeholder__/mypackage" + status, _, fake_stdout = create_build_status(is_tty=False, filter_padding=filter_padding) + spec = MockSpec("mypackage", "1.0", prefix=padded_prefix) + status.add_build(spec, explicit=True, control_w_conn=MockConnection()) + build_id = spec.dag_hash() + status.update_state(build_id, "finished") + output = fake_stdout.getvalue() + common = f"[+] {spec.dag_hash(7)} {spec.name}@{spec.version}" + if filter_padding: + assert output == f"{common} /base/[padded-to-59-chars]/mypackage\n" + else: + assert output == f"{common} {padded_prefix}\n" + class TestSearchFilteringIntegration: """Test search mode with display filtering""" @@ -1048,6 +1221,23 @@ def test_empty_build_list(self): assert "Progress:" in output assert "0/0" in output + def test_no_header_with_finalize(self): + """Test that we don't print a header with finalize=True""" + status, _, fake_stdout = create_build_status(total=2, color=False) + spec_a, spec_b = add_mock_builds(status, 2) + status.update_state(spec_a.dag_hash(), "finished") + status.update_state(spec_b.dag_hash(), "failed") + status.update(finalize=True) + + output = fake_stdout.getvalue() + + # Should not contain header + assert "Progress:" not in output + + # Should contain final status lines for both builds + assert f"[+] {spec_a.dag_hash(7)} {spec_a.name}@{spec_a.version}" in output + assert f"[x] {spec_b.dag_hash(7)} {spec_b.name}@{spec_b.version}" in output + def test_all_builds_finished(self): """Test when all builds are finished""" status, fake_time, _ = create_build_status(total=2) @@ -1080,3 +1270,260 @@ def test_update_progress_rounds_correctly(self): status.update_progress(build_id, 3, 3) assert status.builds[build_id].progress_percent == 100 + + +class TestBuildStatusVerbose: + """Tests for verbose non-TTY log tracking in BuildStatus.""" + + def test_verbose_tracks_first_build(self): + """First add_build() in verbose non-TTY mode sets tracked_build_id and enables echoing.""" + bs, _, _ = create_build_status(is_tty=False, verbose=True, total=4) + spec = MockSpec("trivial-install-test-package", "1.0") + + r_conn, w_conn = Pipe(duplex=False) + + with r_conn, w_conn: + bs.add_build(spec, explicit=True, control_w_conn=w_conn) + + assert bs.tracked_build_id == spec.dag_hash() + written = os.read(r_conn.fileno(), 1) + assert written == b"1" + + def test_verbose_does_not_track_when_already_tracking(self): + """Second add_build() while already tracking does not switch tracking.""" + bs, _, _ = create_build_status(is_tty=False, verbose=True, total=4) + spec1 = MockSpec("pkg1", "1.0") + spec2 = MockSpec("pkg2", "1.0") + + r1, w1 = Pipe(duplex=False) + r2, w2 = Pipe(duplex=False) + with r1, w1, r2, w2: + bs.add_build(spec1, explicit=True, control_w_conn=w1) + first_tracked = bs.tracked_build_id + + bs.add_build(spec2, explicit=False, control_w_conn=w2) + assert bs.tracked_build_id == first_tracked + assert bs.tracked_build_id == spec1.dag_hash() + + # Second build should not have received b"1" + assert not r2.poll(), "Second build should not be enabled" + + def test_verbose_switches_on_finish(self): + """After the tracked build finishes, tracked_build_id is cleared.""" + bs, _, _ = create_build_status(is_tty=False, verbose=True, total=4) + spec = MockSpec("trivial-install-test-package", "1.0") + + r_conn, w_conn = Pipe(duplex=False) + + with r_conn, w_conn: + bs.add_build(spec, explicit=True, control_w_conn=w_conn) + assert bs.tracked_build_id == spec.dag_hash() + + bs.update_state(spec.dag_hash(), "finished") + assert bs.tracked_build_id == "" + + def test_verbose_print_logs_tracked(self): + """print_logs() for the tracked build writes to stdout.""" + bs, _, stdout = create_build_status(is_tty=False, verbose=True, total=1) + spec = MockSpec("trivial-install-test-package", "1.0") + + r_conn, w_conn = Pipe(duplex=False) + + with r_conn, w_conn: + bs.add_build(spec, explicit=True, control_w_conn=w_conn) + bs.print_logs(spec.dag_hash(), b"hello log\n") + + stdout.flush() + assert stdout.buffer.getvalue() == b"hello log\n" + + def test_verbose_print_logs_untracked(self): + """print_logs() for an untracked build discards data.""" + bs, _, stdout = create_build_status(is_tty=False, verbose=True, total=2) + spec1 = MockSpec("pkg1", "1.0") + spec2 = MockSpec("pkg2", "1.0") + + r1, w1 = Pipe(duplex=False) + + with r1, w1: + bs.add_build(spec1, explicit=True, control_w_conn=w1) + bs.add_build(spec2, explicit=False, control_w_conn=None) + + # Only spec1 is tracked; spec2 logs should be discarded + bs.print_logs(spec2.dag_hash(), b"ignored\n") + + stdout.flush() + assert stdout.buffer.getvalue() == b"" + + def test_verbose_tty_no_effect(self): + """In TTY mode, add_build() does not set tracked_build_id automatically.""" + bs, _, _ = create_build_status(is_tty=True, verbose=True, total=4) + spec = MockSpec("trivial-install-test-package", "1.0") + + r_conn, w_conn = Pipe(duplex=False) + + with r_conn, w_conn: + bs.add_build(spec, explicit=True, control_w_conn=w_conn) + assert bs.tracked_build_id == "" + + +class TestBuildStatusColor: + """Tests that BuildStatus respects the explicit color=True/False parameter.""" + + def test_non_tty_finished_color_true_emits_green(self): + """color=True in non-TTY mode: finished line has per-component ANSI colors.""" + spec = MockSpec("pkg", "1.0") + status, _, stdout = create_build_status(is_tty=False, total=1, color=True) + status.add_build(spec, explicit=True) + status.update_state(spec.dag_hash(), "finished") + # green indicator, reset, dark-gray hash + assert stdout.getvalue().startswith("\033[32m[+]\033[0m \033[0;90m") + + def test_non_tty_failed_color_true_emits_red(self): + """color=True in non-TTY mode: failed line has per-component ANSI colors.""" + spec = MockSpec("pkg", "1.0") + status, _, stdout = create_build_status(is_tty=False, total=1, color=True) + status.add_build(spec, explicit=True) + status.update_state(spec.dag_hash(), "failed") + # red indicator, reset, dark-gray hash + assert stdout.getvalue().startswith("\033[31m[x]\033[0m \033[0;90m") + + def test_non_tty_finished_color_false_no_ansi(self): + """color=False in non-TTY mode: finished line has no ANSI escape codes.""" + spec = MockSpec("pkg", "1.0") + status, _, stdout = create_build_status(is_tty=False, total=1, color=False) + status.add_build(spec, explicit=True) + status.update_state(spec.dag_hash(), "finished") + assert "\033[" not in stdout.getvalue() + + +class TestTargetJobs: + """Test set_jobs and its effect on the header.""" + + def test_set_jobs_marks_dirty(self): + """set_jobs with a new value should update target_jobs and mark dirty.""" + status, _, _ = create_build_status() + status.dirty = False + status.set_jobs(3, 2) + assert status.actual_jobs == 3 + assert status.target_jobs == 2 + assert status.dirty is True + status.set_jobs(2, 2) + assert status.actual_jobs == 2 + assert status.target_jobs == 2 + + def test_set_jobs_same_value_no_dirty(self): + """set_jobs with the same value should not mark dirty.""" + status, _, _ = create_build_status() + status.set_jobs(5, 5) + status.dirty = False + status.set_jobs(5, 5) + assert status.dirty is False + + def test_header_shows_target_jobs(self): + """The rendered header should contain the target_jobs count and the word 'jobs'.""" + status, _, fake_stdout = create_build_status(total=1) + add_mock_builds(status, 1) + status.set_jobs(4, 4) + status.update() + output = fake_stdout.getvalue() + assert "4" in output + assert "jobs" in output + + def test_header_shows_arrow_when_pending(self): + """When actual != target, the header should show 'actual=>target jobs'.""" + status, _, fake_stdout = create_build_status(total=1) + add_mock_builds(status, 1) + status.set_jobs(4, 2) + status.update() + output = fake_stdout.getvalue() + assert "4=>2" in output + + +class TestHeadlessMode: + """Test that headless mode suppresses terminal output.""" + + def test_update_suppressed_when_headless(self): + """update() should not write anything when headless is True.""" + status, time_values, stdout = create_build_status(is_tty=True, total=1) + add_mock_builds(status, 1) + status.headless = True + time_values.append(10.0) + status.update() + assert stdout.getvalue() == "" + + def test_print_logs_suppressed_when_headless(self): + """print_logs() should discard data when headless is True.""" + status, _, stdout = create_build_status(is_tty=True, total=1) + specs = add_mock_builds(status, 1) + status.tracked_build_id = specs[0].dag_hash() + status.headless = True + status.print_logs(specs[0].dag_hash(), b"hello world\n") + assert stdout.getvalue() == "" + + def test_update_state_non_tty_suppressed_when_headless(self): + """update_state() non-TTY output should be suppressed when headless.""" + status, _, stdout = create_build_status(is_tty=False, total=1) + spec = MockSpec("pkg", "1.0") + status.add_build(spec, explicit=True) + status.headless = True + stdout.clear() + status.update_state(spec.dag_hash(), "finished") + assert stdout.getvalue() == "" + + def test_update_works_after_headless_cleared(self): + """update() should work normally once headless is cleared.""" + status, time_values, stdout = create_build_status(is_tty=True, total=1, color=False) + add_mock_builds(status, 1) + status.headless = True + time_values.append(10.0) + status.update() + assert stdout.getvalue() == "" + # Clear headless and verify output resumes + status.headless = False + status.dirty = True + status.update() + assert "[/] pkg0 pkg0@0.0 starting" in stdout.getvalue() + + +class TestStdinReader: + def test_basic_ascii(self): + r, w = os.pipe() + try: + reader = StdinReader(r) + os.write(w, b"abc") + assert reader.read() == "abc" + finally: + os.close(r) + os.close(w) + + def test_ansi_stripping(self): + r, w = os.pipe() + try: + reader = StdinReader(r) + os.write(w, b"hello\x1b[Aworld\x1b[B!") + assert reader.read() == "helloworld!" + finally: + os.close(r) + os.close(w) + + def test_multibyte_utf8(self): + r, w = os.pipe() + try: + reader = StdinReader(r) + encoded = "é".encode("utf-8") # 0xc3 0xa9 + os.write(w, encoded[:1]) + # First read: incomplete char, decoder buffers it + result1 = reader.read() + os.write(w, encoded[1:]) + result2 = reader.read() + assert result1 + result2 == "é" + finally: + os.close(r) + os.close(w) + + def test_oserror_returns_empty(self): + r, w = os.pipe() + os.close(w) + os.close(r) + reader = StdinReader(r) + assert reader.read() == "" diff --git a/lib/spack/spack/test/jobserver.py b/lib/spack/spack/test/jobserver.py index 9dff1066ec998c..49c4299fbe6630 100644 --- a/lib/spack/spack/test/jobserver.py +++ b/lib/spack/spack/test/jobserver.py @@ -127,6 +127,10 @@ def test_returns_none_for_missing_fifo(self, tmp_path: pathlib.Path): assert result is None +#: Constant that's larger than the number of jobs used in tests. +ALL_TOKENS = 100 + + class TestJobServer: """Test JobServer class functionality.""" @@ -275,3 +279,167 @@ def test_connection_objects_exist(self): assert js.w_conn is not None and js.w_conn.fileno() == js.w finally: js.close() + + def test_close_warns_when_spack_holds_tokens(self): + """Should warn when Spack closes the jobserver while still holding acquired tokens.""" + js = JobServer(4) + js.acquire(1) # Spack acquires a token without releasing it + with pytest.warns(UserWarning, match="Spack failed to release jobserver tokens"): + js.close() + + def test_close_warns_when_subprocess_holds_tokens(self): + """Should warn when a subprocess acquired a token but never released it.""" + js1 = JobServer(4) + os.read(js1.r, 1) # A subprocess acquires a token without releasing it + with pytest.warns(UserWarning, match="1 jobserver token was not released"): + js1.close() + + js2 = JobServer(4) + os.read(js2.r, 2) # A subprocess acquires two tokens without releasing them + with pytest.warns(UserWarning, match="2 jobserver tokens were not released"): + js2.close() + + def test_has_target_parallelism(self): + """has_target_parallelism() should be True initially.""" + js = JobServer(4) + try: + assert js.has_target_parallelism() is True + js.target_jobs = js.num_jobs - 1 + assert js.has_target_parallelism() is False + finally: + js.close() + + def test_increase_parallelism_not_created(self): + """increase_parallelism() should be a no-op when not self.created.""" + # Simulate an externally attached jobserver by patching created after construction. + js = JobServer(3) + try: + original_num = js.num_jobs + original_target = js.target_jobs + js.created = False + js.increase_parallelism() + assert js.num_jobs == original_num + assert js.target_jobs == original_target + js.decrease_parallelism() + assert js.num_jobs == original_num + assert js.target_jobs == original_target + finally: + js.created = True # restore so close() works + js.close() + + def test_increase_parallelism(self): + """increase_parallelism() should increment num_jobs and target_jobs and add a token.""" + js = JobServer(3) + try: + original_num = js.num_jobs + original_target = js.target_jobs + js.increase_parallelism() + assert js.num_jobs == original_num + 1 + assert js.target_jobs == original_target + 1 + # Verify the "js.num_jobs - 1 tokens in the pipe" invariant. + assert js.acquire(ALL_TOKENS) + 1 == js.num_jobs + finally: + js.close() + + def test_decrease_parallelism_at_floor(self): + """decrease_parallelism() should not go below target_jobs == 1.""" + js = JobServer(1) + try: + # target_jobs starts at 1 + assert js.target_jobs == 1 + js.decrease_parallelism() + assert js.target_jobs == 1 + finally: + js.close() + + def test_decrease_parallelism_token_available(self): + """When pipe has tokens, decrease_parallelism discards one immediately.""" + js = JobServer(3) + try: + # 3-job server starts with 2 tokens in the pipe. + original_num = js.num_jobs + js.decrease_parallelism() + assert js.target_jobs == original_num - 1 + assert js.num_jobs == original_num - 1 + assert js.acquire(ALL_TOKENS) + 1 == js.num_jobs + finally: + js.close() + + def test_decrease_parallelism_no_token_available(self): + """When all tokens are held, decrease_parallelism defers the discard. + A subsequent increase cancels the pending decrease instead of adding a token.""" + js = JobServer(3) + try: + # Drain the pipe so no tokens are available for immediate discard. + assert js.acquire(ALL_TOKENS) == js.num_jobs - 1 + original_num = js.num_jobs + js.decrease_parallelism() + # target_jobs decremented but num_jobs unchanged (no token to discard yet). + assert js.target_jobs == original_num - 1 + assert js.num_jobs == original_num + # increase should cancel the pending decrease, not write a new token. + js.increase_parallelism() + assert js.target_jobs == original_num + assert js.num_jobs == original_num + finally: + js.close() + + def test_maybe_discard_tokens_noop_at_target(self): + """maybe_discard_tokens() should be a no-op when num_jobs == target_jobs.""" + js = JobServer(3) + try: + original_num = js.num_jobs + js.maybe_discard_tokens() # to_discard == 0 + assert js.num_jobs == original_num + finally: + js.close() + + def test_maybe_discard_tokens_discards_when_available(self): + """maybe_discard_tokens() should consume tokens from the pipe.""" + js = JobServer(4) + try: + # Manually set target lower to create a discard requirement. + js.target_jobs = js.num_jobs - 2 + original_num = js.num_jobs + js.maybe_discard_tokens() + assert js.num_jobs < original_num + finally: + js.close() + + def test_maybe_discard_tokens_noop_on_blocking(self): + """maybe_discard_tokens() should not raise when pipe is empty.""" + js = JobServer(3) + try: + # Drain all tokens from the pipe (simulates subprocesses holding them). + assert js.acquire(ALL_TOKENS) == js.num_jobs - 1 + original_num = js.num_jobs + # Artificially lower target so a discard is requested, but pipe is empty. + js.target_jobs = js.num_jobs - 1 + js.maybe_discard_tokens() # Should not raise; num_jobs unchanged. + assert js.num_jobs == original_num + finally: + js.close() + + def test_release_discards_token_when_target_below_num(self): + """release() should discard a token (not return it) when target_jobs < num_jobs.""" + js = JobServer(4) + try: + # Acquire a token. + assert js.acquire(1) == 1 + assert js.tokens_acquired == 1 + # Manually lower target to simulate a pending decrease. + js.target_jobs = js.num_jobs - 1 + original_num = js.num_jobs + # Drain the free tokens from the pipe so we can count them after. + drained = os.read(js.r, ALL_TOKENS) + # Release should discard the token (decrement num_jobs) instead of writing to pipe. + js.release() + assert js.tokens_acquired == 0 + assert js.num_jobs == original_num - 1 + # Pipe should remain empty (nothing written back). + with pytest.raises(BlockingIOError): + os.read(js.r, 1) + finally: + # Restore drained tokens so close() can clean up cleanly. + os.write(js.w, drained) + js.close() diff --git a/lib/spack/spack/test/llnl/url.py b/lib/spack/spack/test/llnl/url.py index cafbde9eae94f5..0c49d51b1d24b3 100644 --- a/lib/spack/spack/test/llnl/url.py +++ b/lib/spack/spack/test/llnl/url.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for spack.llnl.url functions""" + import itertools import pytest diff --git a/lib/spack/spack/test/llnl/util/filesystem.py b/lib/spack/spack/test/llnl/util/filesystem.py index 408303cda1b8af..fc3a485c4799ec 100644 --- a/lib/spack/spack/test/llnl/util/filesystem.py +++ b/lib/spack/spack/test/llnl/util/filesystem.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for ``llnl/util/filesystem.py``""" + import filecmp import os import pathlib diff --git a/lib/spack/spack/test/llnl/util/lang.py b/lib/spack/spack/test/llnl/util/lang.py index 7effe85ebd0b83..d75c7c70dd8657 100644 --- a/lib/spack/spack/test/llnl/util/lang.py +++ b/lib/spack/spack/test/llnl/util/lang.py @@ -11,7 +11,14 @@ import pytest import spack.llnl.util.lang -from spack.llnl.util.lang import dedupe, match_predicate, memoized, pretty_date +from spack.llnl.util.lang import ( + Singleton, + SingletonInstantiationError, + dedupe, + match_predicate, + memoized, + pretty_date, +) @pytest.fixture() @@ -126,6 +133,15 @@ def test_pretty_seconds(): assert spack.llnl.util.lang.pretty_seconds(2.1 / 1000 / 1000 / 1000 / 10) == "0.210ns" +def test_pretty_duration(): + assert spack.llnl.util.lang.pretty_duration(0) == "0s" + assert spack.llnl.util.lang.pretty_duration(45) == "45s" + assert spack.llnl.util.lang.pretty_duration(60) == "1m00s" + assert spack.llnl.util.lang.pretty_duration(125) == "2m05s" + assert spack.llnl.util.lang.pretty_duration(3600) == "1h00m" + assert spack.llnl.util.lang.pretty_duration(3661) == "1h01m" + + def test_match_predicate(): matcher = match_predicate(lambda x: True) assert matcher("foo") @@ -332,6 +348,29 @@ def test_fnmatch_multiple(): assert not regex.match("libbaz.so") +def _attr_error_factory(): + raise AttributeError("Could not make something") + + +def test_singleton_instantiation_attr_failure(): + """ + If an AttributeError occurs during the instantiation of a Singleton + object, we want to see that error. + """ + x = Singleton(_attr_error_factory) + with pytest.raises(SingletonInstantiationError) as last_exception: + x.something + + def follow_exceptions(e): + while e: + yield e + e = e.__cause__ or e.__context__ + + assert any( + "Could not make something" in str(e) for e in follow_exceptions(last_exception.value) + ) + + class TestPriorityOrderedMapping: @pytest.mark.parametrize( "elements,expected", diff --git a/lib/spack/spack/test/llnl/util/link_tree.py b/lib/spack/spack/test/llnl/util/link_tree.py index d6e248b3d88a2d..9481e9e329336b 100644 --- a/lib/spack/spack/test/llnl/util/link_tree.py +++ b/lib/spack/spack/test/llnl/util/link_tree.py @@ -19,7 +19,8 @@ visit_directory_tree, working_dir, ) -from spack.llnl.util.link_tree import DestinationMergeVisitor, LinkTree, SourceMergeVisitor +from spack.llnl.util.link_tree import DestinationMergeVisitor, LinkTree, MultiPrefixMerger +from spack.test.conftest import FsTree @pytest.fixture @@ -219,8 +220,7 @@ def test_source_merge_visitor_does_not_follow_symlinked_dirs_at_depth(tmp_path: with open(j("a", "b", "c", "d", "file"), "wb"): pass - visitor = SourceMergeVisitor() - visit_directory_tree(str(tmp_path), visitor) + visitor = MultiPrefixMerger([tmp_path]) assert [p for p in visitor.files.keys()] == [ j("a", "b", "c", "d", "file"), j("a", "b", "c", "symlink_d"), # treated as a file, not expanded @@ -263,8 +263,7 @@ def test_source_merge_visitor_cant_be_cyclical(tmp_path: pathlib.Path): symlink(j("symlink_b"), j("a", "symlink_b_b")) symlink(j("..", "a"), j("b", "symlink_a")) - visitor = SourceMergeVisitor() - visit_directory_tree(str(tmp_path), visitor) + visitor = MultiPrefixMerger([tmp_path]) assert [p for p in visitor.files.keys()] == [ j("a", "symlink_b"), j("a", "symlink_b_b"), @@ -297,8 +296,7 @@ def test_destination_merge_visitor_always_errors_on_symlinked_dirs(tmp_path: pat pass os.symlink("..", "example_b") - visitor = SourceMergeVisitor() - visit_directory_tree(str(src_path), visitor) + visitor = MultiPrefixMerger([src_path]) visit_directory_tree(str(dst_path), DestinationMergeVisitor(visitor)) assert visitor.fatal_conflicts @@ -321,14 +319,12 @@ def test_destination_merge_visitor_file_dir_clashes(tmp_path: pathlib.Path): with open("example", "wb"): pass - a_to_b = SourceMergeVisitor() - visit_directory_tree(str(a_path), a_to_b) + a_to_b = MultiPrefixMerger([a_path]) visit_directory_tree(str(b_path), DestinationMergeVisitor(a_to_b)) assert a_to_b.fatal_conflicts assert a_to_b.fatal_conflicts[0].dst == "example" - b_to_a = SourceMergeVisitor() - visit_directory_tree(str(b_path), b_to_a) + b_to_a = MultiPrefixMerger([b_path]) visit_directory_tree(str(a_path), DestinationMergeVisitor(b_to_a)) assert b_to_a.fatal_conflicts assert b_to_a.fatal_conflicts[0].dst == "example" @@ -355,15 +351,15 @@ def u(path: str) -> str: (tmp_path / "b" / u("dir")).symlink_to(tmp_path / "a" / "dir") (tmp_path / "b" / "bar").write_bytes(b"hello") - visitor_1 = SourceMergeVisitor(normalize_paths=normalize) - visitor_1.set_projection(str(tmp_path / "view")) - for p in ("a", "b"): - visit_directory_tree(str(tmp_path / p), visitor_1) + visitor_1 = MultiPrefixMerger( + [(tmp_path / "a", tmp_path / "view"), (tmp_path / "b", tmp_path / "view")], + normalize_paths=normalize, + ) - visitor_2 = SourceMergeVisitor(normalize_paths=normalize) - visitor_2.set_projection(str(tmp_path / "view")) - for p in ("b", "a"): - visit_directory_tree(str(tmp_path / p), visitor_2) + visitor_2 = MultiPrefixMerger( + [(tmp_path / "b", tmp_path / "view"), (tmp_path / "a", tmp_path / "view")], + normalize_paths=normalize, + ) assert not visitor_1.file_conflicts and not visitor_2.file_conflicts assert not visitor_1.fatal_conflicts and not visitor_2.fatal_conflicts @@ -388,11 +384,9 @@ def test_source_merge_visitor_deals_with_dangling_symlinks(tmp_path: pathlib.Pat (tmp_path / "dir_b").mkdir() (tmp_path / "dir_b" / "file").write_bytes(b"data") - visitor = SourceMergeVisitor() - visitor.set_projection(str(tmp_path / "view")) - - visit_directory_tree(str(tmp_path / "dir_a"), visitor) - visit_directory_tree(str(tmp_path / "dir_b"), visitor) + visitor = MultiPrefixMerger( + [(tmp_path / "dir_a", tmp_path / "view"), (tmp_path / "dir_b", tmp_path / "view")] + ) # Check that a conflict was registered. assert len(visitor.file_conflicts) == 1 @@ -412,9 +406,7 @@ def test_source_visitor_file_file(tmp_path: pathlib.Path, normalize: bool): (tmp_path / "a" / "file").write_bytes(b"") (tmp_path / "b" / "FILE").write_bytes(b"") - v = SourceMergeVisitor(normalize_paths=normalize) - for p in ("a", "b"): - visit_directory_tree(str(tmp_path / p), v) + v = MultiPrefixMerger([tmp_path / "a", tmp_path / "b"], normalize_paths=normalize) if normalize: assert len(v.files) == 1 @@ -435,12 +427,8 @@ def test_source_visitor_file_dir(tmp_path: pathlib.Path, normalize: bool): (tmp_path / "a" / "file").write_bytes(b"") (tmp_path / "b").mkdir() (tmp_path / "b" / "FILE").mkdir() - v1 = SourceMergeVisitor(normalize_paths=normalize) - for p in ("a", "b"): - visit_directory_tree(str(tmp_path / p), v1) - v2 = SourceMergeVisitor(normalize_paths=normalize) - for p in ("b", "a"): - visit_directory_tree(str(tmp_path / p), v2) + v1 = MultiPrefixMerger([tmp_path / "a", tmp_path / "b"], normalize_paths=normalize) + v2 = MultiPrefixMerger([tmp_path / "b", tmp_path / "a"], normalize_paths=normalize) assert not v1.file_conflicts and not v2.file_conflicts @@ -460,9 +448,7 @@ def test_source_visitor_dir_dir(tmp_path: pathlib.Path, normalize: bool): (tmp_path / "a" / "dir").mkdir() (tmp_path / "b").mkdir() (tmp_path / "b" / "DIR").mkdir() - v = SourceMergeVisitor(normalize_paths=normalize) - for p in ("a", "b"): - visit_directory_tree(str(tmp_path / p), v) + v = MultiPrefixMerger([tmp_path / "a", tmp_path / "b"], normalize_paths=normalize) assert not v.files assert not v.fatal_conflicts @@ -483,8 +469,7 @@ def test_dst_visitor_file_file(tmp_path: pathlib.Path, normalize: bool): (tmp_path / "a" / "file").write_bytes(b"") (tmp_path / "b" / "FILE").write_bytes(b"") - src = SourceMergeVisitor(normalize_paths=normalize) - visit_directory_tree(str(tmp_path / "a"), src) + src = MultiPrefixMerger([tmp_path / "a"], normalize_paths=normalize) visit_directory_tree(str(tmp_path / "b"), DestinationMergeVisitor(src)) assert len(src.files) == 1 @@ -505,11 +490,9 @@ def test_dst_visitor_file_dir(tmp_path: pathlib.Path, normalize: bool): (tmp_path / "a" / "file").write_bytes(b"") (tmp_path / "b").mkdir() (tmp_path / "b" / "FILE").mkdir() - src1 = SourceMergeVisitor(normalize_paths=normalize) - visit_directory_tree(str(tmp_path / "a"), src1) + src1 = MultiPrefixMerger([tmp_path / "a"], normalize_paths=normalize) visit_directory_tree(str(tmp_path / "b"), DestinationMergeVisitor(src1)) - src2 = SourceMergeVisitor(normalize_paths=normalize) - visit_directory_tree(str(tmp_path / "b"), src2) + src2 = MultiPrefixMerger([tmp_path / "b"], normalize_paths=normalize) visit_directory_tree(str(tmp_path / "a"), DestinationMergeVisitor(src2)) assert len(src1.files) == 1 @@ -527,3 +510,103 @@ def test_dst_visitor_file_dir(tmp_path: pathlib.Path, normalize: bool): else: assert not src1.fatal_conflicts and not src2.fatal_conflicts assert not src1.file_conflicts and not src2.file_conflicts + + +def test_unique_subdir_optimization(tmp_path: pathlib.Path): + """A subdirectory at depth > 0 unique to one prefix should be registered as a single file + entry (to be symlinked as a directory) in symlink mode, not recursed into. Top-level (depth 0) + dirs are always recursed into.""" + src_a = tmp_path / "a" + src_b = tmp_path / "b" + + FsTree( + tmp_path, + { + # shared dir: lib (exists in both) -- depth 0, shared + "a/lib/liba.so": FsTree.file(b"a"), + "b/lib/libb.so": FsTree.file(b"b"), + # shared dir: share (exists in both) -- depth 0, shared + # but unique subdirs at depth 1 + "a/share/app_a/data.txt": FsTree.file(b"a"), + "a/share/app_a/sub/deep.txt": FsTree.file(b"deep"), + "b/share/app_b/info.txt": FsTree.file(b"b"), + # unique dir: include (only in a) -- depth 0, unique but NOT collapsed + "a/include/a.h": FsTree.file(b"a"), + # unique dir: bin (only in b) -- depth 0, unique but NOT collapsed + "b/bin/prog": FsTree.file(b"p"), + }, + ) + + visitor = MultiPrefixMerger(sources=[src_a, src_b], dir_symlink_optimization=True) + + assert not visitor.fatal_conflicts + assert not visitor.file_conflicts + + # depth 0 unique dirs should be recursed into (directories), not collapsed + assert "include" in visitor.directories + assert "include" not in visitor.files + assert os.path.join("include", "a.h") in visitor.files + assert "bin" in visitor.directories + assert "bin" not in visitor.files + assert os.path.join("bin", "prog") in visitor.files + + # "lib" should be a directory (shared), with individual files inside + assert "lib" in visitor.directories + assert os.path.join("lib", "liba.so") in visitor.files + assert os.path.join("lib", "libb.so") in visitor.files + + # depth 1 unique subdirs under shared parent should be collapsed (dir-level symlinks) + assert "share" in visitor.directories + assert os.path.join("share", "app_a") in visitor.files + assert visitor.files[os.path.join("share", "app_a")] == ( + str(src_a), + os.path.join("share", "app_a"), + ) + assert os.path.join("share", "app_b") in visitor.files + assert visitor.files[os.path.join("share", "app_b")] == ( + str(src_b), + os.path.join("share", "app_b"), + ) + + # Subdirs of collapsed dirs should NOT appear in directories + assert os.path.join("share", "app_a") not in visitor.directories + assert os.path.join("share", "app_a", "sub") not in visitor.directories + assert os.path.join("share", "app_b") not in visitor.directories + + +def test_unique_subdir_optimization_disabled(tmp_path: pathlib.Path): + """For hardlink/copy views, unique subdirs should NOT be dir-level symlinks; + individual files should be registered instead.""" + src_a = tmp_path / "a" + src_b = tmp_path / "b" + + FsTree(tmp_path, {"a/lib/a/liba.so": FsTree.file(b"a"), "b/lib/b/libb.so": FsTree.file(b"b")}) + + visitor = MultiPrefixMerger(sources=[src_a, src_b], dir_symlink_optimization=False) + + assert not visitor.fatal_conflicts + assert not visitor.file_conflicts + + # Check that all files are there + assert "lib" in visitor.directories + assert os.path.join("lib", "a") in visitor.directories + assert os.path.join("lib", "b") in visitor.directories + assert os.path.join("lib", "a", "liba.so") in visitor.files + assert os.path.join("lib", "b", "libb.so") in visitor.files + + # No dirs are symlinked. + assert os.path.join("lib", "a") not in visitor.files + assert os.path.join("lib", "b") not in visitor.files + + +def test_projection_dirs_created(tmp_path: pathlib.Path): + """Projection directories should be registered.""" + src_a = tmp_path / "a" + + FsTree(tmp_path, {"a/file.txt": FsTree.file(b"a")}) + + visitor = MultiPrefixMerger(sources=[(src_a, "proj/sub")], dir_symlink_optimization=True) + + assert "proj" in visitor.directories + assert os.path.join("proj", "sub") in visitor.directories + assert os.path.join("proj", "sub", "file.txt") in visitor.files diff --git a/lib/spack/spack/test/llnl/util/lock.py b/lib/spack/spack/test/llnl/util/lock.py index 26262132100e26..a7b3f9aa6da651 100644 --- a/lib/spack/spack/test/llnl/util/lock.py +++ b/lib/spack/spack/test/llnl/util/lock.py @@ -4,23 +4,9 @@ """These tests ensure that our lock works correctly. -This can be run in two ways. +Run with pytest:: -First, it can be run as a node-local test, with a typical invocation like -this:: - - spack test lock - -You can *also* run it as an MPI program, which allows you to test locks -across nodes. So, e.g., you can run the test like this:: - - mpirun -n 7 spack test lock - -And it will test locking correctness among MPI processes. Ideally, you -want the MPI processes to span across multiple nodes, so, e.g., for Slurm -you might do this:: - - srun -N 7 -n 7 -m cyclic spack test lock + pytest lib/spack/spack/test/llnl/util/lock.py You can use this to test whether your shared filesystem properly supports POSIX reader-writer locking with byte ranges through fcntl. @@ -36,15 +22,15 @@ Add names and paths for your preferred filesystem mounts to test on them; the tests are parametrized to run on all the filesystems listed in this -dict. Note that 'tmp' will be skipped for MPI testing, as it is often a -node-local filesystem, and multi-node tests will fail if the locks aren't -actually on a shared filesystem. +dict. """ + import collections import errno import getpass import glob +import multiprocessing import os import pathlib import shutil @@ -645,22 +631,52 @@ def test_upgrade_read_to_write(private_lock_path): lock.acquire_read() assert lock._reads == 1 assert lock._writes == 0 - assert lock._file.mode == "rb+" + assert lock._file_ref.fh.mode == "rb+" lock.acquire_write() assert lock._reads == 1 assert lock._writes == 1 - assert lock._file.mode == "rb+" + assert lock._file_ref.fh.mode == "rb+" + + lock.release_write() + assert lock._reads == 1 + assert lock._writes == 0 + assert lock._file_ref.fh.mode == "rb+" + + lock.release_read() + assert lock._reads == 0 + assert lock._writes == 0 + assert not lock._file_ref.fh.closed # recycle the file handle for next lock + +def test_release_write_downgrades_to_shared(private_lock_path): + """Releasing a write lock while a read lock is held must downgrade the POSIX lock + from exclusive to shared, allowing other processes to acquire read locks.""" + lock = lk.Lock(private_lock_path) + lock.acquire_read() + lock.acquire_write() lock.release_write() assert lock._reads == 1 assert lock._writes == 0 - assert lock._file.mode == "rb+" + + ctx = multiprocessing.get_context() + q = ctx.Queue() + + # Another process must be able to acquire a shared read lock concurrently. + p = ctx.Process(target=_child_try_acquire_read, args=(private_lock_path, q)) + p.start() + p.join() + assert q.get() is True + + # But must not be able to acquire an exclusive write lock. + p = ctx.Process(target=_child_try_acquire_write, args=(private_lock_path, q)) + p.start() + p.join() + assert q.get() is False lock.release_read() assert lock._reads == 0 assert lock._writes == 0 - assert lock._file is None @pytest.mark.skipif(getuid() == 0, reason="user is root") @@ -678,15 +694,12 @@ def test_upgrade_read_to_write_fails_with_readonly_file(private_lock_path): lock.acquire_read() assert lock._reads == 1 assert lock._writes == 0 - assert lock._file.mode == "rb" + assert lock._file_ref.fh.mode == "rb" # upgrade to write here with pytest.raises(lk.LockROFileError): lock.acquire_write() - # TODO: lk.FILE_TRACKER does not release private_lock_path - lk.FILE_TRACKER.release_by_stat(os.stat(private_lock_path)) - class ComplexAcquireAndRelease: def __init__(self, lock_path): @@ -963,121 +976,6 @@ def exit_fn(t, v, tb): assert vals["exception"] -@pytest.mark.parametrize( - "transaction,type", [(lk.ReadTransaction, "read"), (lk.WriteTransaction, "write")] -) -def test_transaction_with_context_manager(lock_path, transaction, type): - class MockLock(AssertLock): - def assert_acquire_read(self): - assert not vals["entered_ctx"] - assert not vals["exited_ctx"] - - def assert_release_read(self): - assert vals["entered_ctx"] - assert vals["exited_ctx"] - - def assert_acquire_write(self): - assert not vals["entered_ctx"] - assert not vals["exited_ctx"] - - def assert_release_write(self): - assert vals["entered_ctx"] - assert vals["exited_ctx"] - - class TestContextManager: - def __enter__(self): - vals["entered_ctx"] = True - - def __exit__(self, t, v, tb): - assert not vals["released_%s" % type] - vals["exited_ctx"] = True - vals["exception_ctx"] = t or v or tb - return exit_ctx_result - - def exit_fn(t, v, tb): - assert not vals["released_%s" % type] - vals["exited_fn"] = True - vals["exception_fn"] = t or v or tb - return exit_fn_result - - exit_fn_result, exit_ctx_result = False, False - vals = collections.defaultdict(lambda: False) - lock = MockLock(lock_path, vals) - - with transaction(lock, acquire=TestContextManager, release=exit_fn): - pass - - assert vals["entered_ctx"] - assert vals["exited_ctx"] - assert vals["exited_fn"] - assert not vals["exception_ctx"] - assert not vals["exception_fn"] - - vals.clear() - with transaction(lock, acquire=TestContextManager): - pass - - assert vals["entered_ctx"] - assert vals["exited_ctx"] - assert not vals["exited_fn"] - assert not vals["exception_ctx"] - assert not vals["exception_fn"] - - # below are tests for exceptions with and without suppression - def assert_ctx_and_fn_exception(raises=True): - vals.clear() - - if raises: - with pytest.raises(Exception): - with transaction(lock, acquire=TestContextManager, release=exit_fn): - raise Exception() - else: - with transaction(lock, acquire=TestContextManager, release=exit_fn): - raise Exception() - - assert vals["entered_ctx"] - assert vals["exited_ctx"] - assert vals["exited_fn"] - assert vals["exception_ctx"] - assert vals["exception_fn"] - - def assert_only_ctx_exception(raises=True): - vals.clear() - - if raises: - with pytest.raises(Exception): - with transaction(lock, acquire=TestContextManager): - raise Exception() - else: - with transaction(lock, acquire=TestContextManager): - raise Exception() - - assert vals["entered_ctx"] - assert vals["exited_ctx"] - assert not vals["exited_fn"] - assert vals["exception_ctx"] - assert not vals["exception_fn"] - - # no suppression - assert_ctx_and_fn_exception(raises=True) - assert_only_ctx_exception(raises=True) - - # suppress exception only in function - exit_fn_result, exit_ctx_result = True, False - assert_ctx_and_fn_exception(raises=False) - assert_only_ctx_exception(raises=True) - - # suppress exception only in context - exit_fn_result, exit_ctx_result = False, True - assert_ctx_and_fn_exception(raises=False) - assert_only_ctx_exception(raises=False) - - # suppress exception in function and context - exit_fn_result, exit_ctx_result = True, True - assert_ctx_and_fn_exception(raises=False) - assert_only_ctx_exception(raises=False) - - def test_nested_write_transaction(lock_path): """Ensure that the outermost write transaction writes.""" @@ -1376,3 +1274,140 @@ def test_upgrade_read_fails(tmp_path: pathlib.Path): with pytest.raises(lk.LockUpgradeError, match=msg): lock.upgrade_read_to_write() lock.release_write() + + +@pytest.mark.parametrize("acquire", ["acquire_write", "acquire_read"]) +def test_acquire_after_fork(tmp_path: pathlib.Path, acquire: str): + """After fork, acquire_write/read must not silently succeed due to inherited counters.""" + try: + ctx = multiprocessing.get_context("fork") + except ValueError: + pytest.skip("fork start method not available on this platform") + + lockfile = str(tmp_path / "lockfile") + lock = lk.Lock(lockfile) + result = ctx.Queue() + + def child(): + assert lock._writes == 1 # due to forking, but POSIX lock is NOT held by this process + try: + if acquire == "acquire_write": + lock.acquire_write(lock_fail_timeout) + elif acquire == "acquire_read": + lock.acquire_read(lock_fail_timeout) + else: + assert False # should never get here + result.put("no_error") + except lk.LockTimeoutError: + result.put("timed_out") + + lock.acquire_write() + try: + p = ctx.Process(target=child) + p.start() + p.join() + assert result.get() == "timed_out" + finally: + lock.release_write() + + +def _child_try_acquire_write(lock_path: str, result_queue): + lock = lk.Lock(lock_path) + result_queue.put(lock.try_acquire_write()) + + +def _child_try_acquire_read(lock_path: str, result_queue): + lock = lk.Lock(lock_path) + result_queue.put(lock.try_acquire_read()) + + +def test_try_acquire_read(tmp_path: pathlib.Path): + """Test non-blocking try_acquire_read.""" + lock = lk.Lock(str(tmp_path / "lockfile")) + + # Succeeds on unlocked lock + assert lock.try_acquire_read() is True + assert lock._reads == 1 + + # Succeeds again (nested) + assert lock.try_acquire_read() is True + assert lock._reads == 2 + + lock.release_read() + lock.release_read() + ctx = multiprocessing.get_context() + + # Fails when another process holds an exclusive write lock + lock.acquire_write() + try: + q = ctx.Queue() + p = ctx.Process(target=_child_try_acquire_read, args=(str(tmp_path / "lockfile"), q)) + p.start() + p.join() + assert q.get() is False + finally: + lock.release_write() + + +def test_try_acquire_write(tmp_path: pathlib.Path): + """Test non-blocking try_acquire_write.""" + lock = lk.Lock(str(tmp_path / "lockfile")) + ctx = multiprocessing.get_context() + + # Succeeds on unlocked lock + assert lock.try_acquire_write() is True + assert lock._writes == 1 + + # Succeeds again (nested) + assert lock.try_acquire_write() is True + assert lock._writes == 2 + + lock.release_write() + lock.release_write() + + # Fails when another process holds a write lock + lock.acquire_write() + try: + q = ctx.Queue() + p = ctx.Process(target=_child_try_acquire_write, args=(str(tmp_path / "lockfile"), q)) + p.start() + p.join() + assert q.get() is False + finally: + lock.release_write() + + # Fails when another process holds a read lock + lock.acquire_read() + try: + q = ctx.Queue() + p = ctx.Process(target=_child_try_acquire_write, args=(str(tmp_path / "lockfile"), q)) + p.start() + p.join() + assert q.get() is False + finally: + lock.release_read() + + +def _child_fails_to_acquire_read(_lock: lk.Lock): + try: + _lock.acquire_read(timeout=1e-9) + except lk.LockTimeoutError: + return + assert False, "Child process should not have been able to acquire read lock" + + +def test_read_after_write_does_not_accidentally_downgrade(tmp_path: pathlib.Path): + """Test that acquiring a read lock after a write lock does not accidentally downgrade the + write lock, by having another process attempt to acquire a read lock.""" + lock = lk.Lock(str(tmp_path / "lockfile")) + lock.acquire_write() + lock.acquire_read() # should not downgrade the write lock + try: + # No matter the start method, the child process shouldn't be able to acquire a read lock. + p = multiprocessing.Process(target=_child_fails_to_acquire_read, args=(lock,)) + p.start() + p.join() + assert p.exitcode == 0 + finally: + lock.release_read() + lock.release_write() diff --git a/lib/spack/spack/test/llnl/util/symlink.py b/lib/spack/spack/test/llnl/util/symlink.py index f9196c105e3c15..69b75e3f71868f 100644 --- a/lib/spack/spack/test/llnl/util/symlink.py +++ b/lib/spack/spack/test/llnl/util/symlink.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for Windows symlink functionality.""" + import os import pathlib import tempfile diff --git a/lib/spack/spack/test/llnl/util/tty/color.py b/lib/spack/spack/test/llnl/util/tty/color.py index 14bef046a3edaf..7f752b6365c412 100644 --- a/lib/spack/spack/test/llnl/util/tty/color.py +++ b/lib/spack/spack/test/llnl/util/tty/color.py @@ -8,6 +8,7 @@ import pytest import spack.llnl.util.tty.color as color +from spack.llnl.util.tty.color import cescape, colorize, csub test_text = [ "@r{The quick brown fox jumps over the lazy yellow dog.", @@ -49,3 +50,23 @@ def test_color_wrap(cols, text, indent): # make sure we wrap the same as textwrap assert color.csub(color_wrapped) == wrapped assert plain_cwrapped == wrapped + + +def test_cescape_at_sign_roundtrip(): + """cescape followed by colorize should not double-escape '@' inside color blocks.""" + raw = 'if spec.satisfies("@:25.1"):' + colorized = colorize("@R{%s}" % cescape(raw), color=True) + assert csub(colorized) == raw + + +def test_cescape_multiple_at_signs_roundtrip(): + """Multiple consecutive '@' characters should survive a cescape/colorize roundtrip.""" + raw = "foo @@@@@bar" + colorized = colorize("@R{%s}" % cescape(raw), color=True) + assert csub(colorized) == raw + + +def test_colorize_top_level_consecutive_escaped_ats(): + """Consecutive @@ at the top level (outside braces) must each unescape independently.""" + assert colorize("@@@@", color=False) == "@@" + assert colorize("@@@@@@", color=False) == "@@@" diff --git a/lib/spack/spack/test/llnl/util/tty/log.py b/lib/spack/spack/test/llnl/util/tty/log.py index 19872dd812553a..5be2d720b36cc4 100644 --- a/lib/spack/spack/test/llnl/util/tty/log.py +++ b/lib/spack/spack/test/llnl/util/tty/log.py @@ -8,8 +8,6 @@ from types import ModuleType from typing import Optional -import pytest - import spack.llnl.util.tty.log as log from spack.llnl.util.filesystem import working_dir from spack.util.executable import Executable @@ -23,9 +21,6 @@ pass -pytestmark = pytest.mark.not_on_windows("does not run on windows") - - @contextlib.contextmanager def nullcontext(): yield @@ -165,4 +160,30 @@ def test_log_subproc_and_echo_output(capfd, tmp_path: pathlib.Path): # Check captured output (echoed content) # Note: 'logged' is not echoed because force_echo() scope ended - assert capfd.readouterr()[0] == "echo\n" + # Note: "print(echo)" above automatically uses an "\r\n" on Windows + # and will replace any \n with \r\n (so end=\n does not work) + # \r\n is expected and correct here + # Note: the above line ending constraint is an artifact of + # pytest's capfd. This is potentially (however unlikely) + # subject to change with future versions of pytest. + # if this test suddenly starts failing, verifying the line + # endings from capfd is a good starting place. + newline = "\r\n" if sys.platform == "win32" else "\n" + assert capfd.readouterr()[0] == f"echo{newline}" + + +def test_nested_logging_contexts(capfd, tmp_path): + with working_dir(str(tmp_path)): + with log.log_output("foo.txt"): + with log.log_output("bar.txt"): + print("inner") + print("outer") + + with open("foo.txt", "r", encoding="utf-8") as f: + log_captured_out = f.read() + assert "outer\n" in log_captured_out + assert "inner\n" not in log_captured_out + with open("bar.txt", "r", encoding="utf-8") as f: + log_captured_out = f.read() + assert "inner\n" in log_captured_out + assert "outer\n" not in log_captured_out diff --git a/lib/spack/spack/test/main.py b/lib/spack/spack/test/main.py index d31a8a442b2cae..ba18643bca7f8b 100644 --- a/lib/spack/spack/test/main.py +++ b/lib/spack/spack/test/main.py @@ -26,6 +26,11 @@ ) +@pytest.fixture(autouse=True) +def _clear_commit_cache(): + spack.get_spack_commit.cache_clear() + + def test_version_git_nonsense_output(tmp_path: pathlib.Path, working_env, monkeypatch): git = tmp_path / "git" with open(git, "w", encoding="utf-8") as f: @@ -62,9 +67,7 @@ def test_git_sha_output(tmp_path: pathlib.Path, working_env, monkeypatch): f.write( """#!/bin/sh echo {0} -""".format( - sha - ) +""".format(sha) ) fs.set_executable(str(git)) @@ -93,6 +96,10 @@ def test_main_calls_get_version(capfd, working_env, monkeypatch): assert spack.spack_version == out.strip() +def test_unrecognized_top_level_flag(): + assert spack.main.main(["-o", "mirror", "list"]) != 0 + + def test_get_version_bad_git(tmp_path: pathlib.Path, working_env, monkeypatch): bad_git = str(tmp_path / "git") with open(bad_git, "w", encoding="utf-8") as f: diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index 33bbc8cac36341..1cb0e73505b551 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -7,6 +7,7 @@ This just tests whether the right args are getting passed to make. """ + import os import pathlib diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index bf4219aeadfccf..e268a608dcc55c 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -18,7 +18,6 @@ import spack.mirrors.utils import spack.patch import spack.stage -import spack.util.spack_json as sjson import spack.util.url as url_util from spack.cmd.common.arguments import mirror_name_or_url from spack.llnl.util.filesystem import resolve_link_target_relative_to_the_link, working_dir @@ -146,8 +145,6 @@ def test_all_mirror(mock_git_repository, mock_svn_repository, mock_hg_repository def test_roundtrip_mirror(mirror: spack.mirrors.mirror.Mirror): mirror_yaml = mirror.to_yaml() assert spack.mirrors.mirror.Mirror.from_yaml(mirror_yaml) == mirror - mirror_json = mirror.to_json() - assert spack.mirrors.mirror.Mirror.from_json(mirror_json) == mirror @pytest.mark.parametrize( @@ -159,58 +156,6 @@ def test_invalid_yaml_mirror(invalid_yaml): assert invalid_yaml in str(e.value) -@pytest.mark.parametrize("invalid_json, error_message", [("{13:", "Expecting property name")]) -def test_invalid_json_mirror(invalid_json, error_message): - with pytest.raises(sjson.SpackJSONError) as e: - spack.mirrors.mirror.Mirror.from_json(invalid_json) - exc_msg = str(e.value) - assert exc_msg.startswith("error parsing JSON mirror:") - assert error_message in exc_msg - - -@pytest.mark.parametrize( - "mirror_collection", - [ - spack.mirrors.mirror.MirrorCollection( - mirrors={ - "example-mirror": spack.mirrors.mirror.Mirror( - "https://example.com/fetch", "https://example.com/push" - ).to_dict() - } - ) - ], -) -def test_roundtrip_mirror_collection(mirror_collection): - mirror_collection_yaml = mirror_collection.to_yaml() - assert ( - spack.mirrors.mirror.MirrorCollection.from_yaml(mirror_collection_yaml) - == mirror_collection - ) - mirror_collection_json = mirror_collection.to_json() - assert ( - spack.mirrors.mirror.MirrorCollection.from_json(mirror_collection_json) - == mirror_collection - ) - - -@pytest.mark.parametrize( - "invalid_yaml", ["playing_playlist: {{ action }} playlist {{ playlist_name }}"] -) -def test_invalid_yaml_mirror_collection(invalid_yaml): - with pytest.raises(SpackYAMLError, match="error parsing YAML") as e: - spack.mirrors.mirror.MirrorCollection.from_yaml(invalid_yaml) - assert invalid_yaml in str(e.value) - - -@pytest.mark.parametrize("invalid_json, error_message", [("{13:", "Expecting property name")]) -def test_invalid_json_mirror_collection(invalid_json, error_message): - with pytest.raises(sjson.SpackJSONError) as e: - spack.mirrors.mirror.MirrorCollection.from_json(invalid_json) - exc_msg = str(e.value) - assert exc_msg.startswith("error parsing JSON mirror collection:") - assert error_message in exc_msg - - def test_mirror_archive_paths_no_version(mock_packages, mock_archive): spec = spack.concretize.concretize_one( Spec("trivial-install-test-package@=nonexistingversion") @@ -273,6 +218,27 @@ def archive(dst): pass +def test_cache_store_atomic_on_failure(tmp_path: pathlib.Path): + """A failed archive() must not leave a partial file at the final destination.""" + + class FailingFetcher: + cachable = True + + @staticmethod + def archive(dst): + with open(dst, "wb") as f: + f.write(b"partial") + raise RuntimeError("simulated failure mid-archive") + + for cache in [ + spack.caches.MirrorCache(root=str(tmp_path), skip_unstable_versions=False), + spack.fetch_strategy.FsCache(str(tmp_path)), + ]: + with pytest.raises(RuntimeError, match="simulated failure"): + cache.store(FailingFetcher(), "pkg/pkg-1.0.tar.gz") + assert not (tmp_path / "pkg" / "pkg-1.0.tar.gz").exists() + + @pytest.mark.regression("14067") def test_mirror_layout_make_alias(tmp_path: pathlib.Path): """Confirm that the cosmetic symlink created in the mirror cache (which may diff --git a/lib/spack/spack/test/modules/common.py b/lib/spack/spack/test/modules/common.py index 87a9083f7f2785..092345abaeecdb 100644 --- a/lib/spack/spack/test/modules/common.py +++ b/lib/spack/spack/test/modules/common.py @@ -116,9 +116,7 @@ def test_upstream_module_index(): {0}: path: /path/to/a use_name: a -""".format( - s1.dag_hash() - ) +""".format(s1.dag_hash()) module_indices = [{"tcl": spack.modules.common._read_module_index(tcl_module_index)}, {}] @@ -154,9 +152,7 @@ def test_get_module_upstream(): {0}: path: /path/to/a use_name: a -""".format( - s1.dag_hash() - ) +""".format(s1.dag_hash()) module_indices = [{}, {"tcl": spack.modules.common._read_module_index(tcl_module_index)}] diff --git a/lib/spack/spack/test/modules/lmod.py b/lib/spack/spack/test/modules/lmod.py index 7bb1a8fd0a6051..7040f3177d7f34 100644 --- a/lib/spack/spack/test/modules/lmod.py +++ b/lib/spack/spack/test/modules/lmod.py @@ -40,12 +40,14 @@ def compiler(request): @pytest.fixture( params=[ - ("mpich@3.0.4", ("mpi",)), - ("mpich@3.0.1", []), - ("openblas@0.2.15", ("blas",)), - ("openblas-with-lapack@0.2.15", ("blas", "lapack")), - ("mpileaks@2.3", ("mpi",)), - ("mpileaks@2.1", []), + ("mpich@3.0.4", ("mpi",), True, False), + ("mpich@3.0.1", [], True, True), + ("openblas@0.2.15", ("blas",), True, False), + ("openblas-with-lapack@0.2.15", ("blas", "lapack"), True, False), + ("mpileaks@2.3", ("mpi",), True, False), + ("mpileaks@2.1", [], True, False), + ("py-extension1@2.0", ("python",), False, True), + ("python@3.8.0", ("python",), False, True), ] ) def provider(request): @@ -69,8 +71,14 @@ def test_layout_for_specs_compiled_with_core_compilers( def test_file_layout(self, compiler, provider, factory, module_configuration): """Tests the layout of files in the hierarchy is the one expected.""" module_configuration("complex_hierarchy") - spec_string, services = provider - module, spec = factory(spec_string + "%" + compiler) + spec_string, services, use_compiler, place_in_core = provider + + # Non-python specs add compiler + factory_string = spec_string + if use_compiler: + factory_string += "%" + compiler + + module, spec = factory(factory_string) layout = module.layout @@ -82,7 +90,8 @@ def test_file_layout(self, compiler, provider, factory, module_configuration): # is transformed to r"Core" if the compiler is listed among core # compilers # Check that specs listed as core_specs are transformed to "Core" - if compiler == "clang@=15.0.0" or spec_string == "mpich@3.0.1": + # Check that specs with no hierarchy components are transformed to "Core" + if "clang@=15.0.0" in factory_string or place_in_core: assert "Core" in layout.available_path_parts else: assert compiler.replace("@=", "/") in layout.available_path_parts @@ -94,11 +103,16 @@ def test_file_layout(self, compiler, provider, factory, module_configuration): # JCSDA fork only - no hashes in service_part # service_part = "-".join([service_part, layout.spec.dag_hash(length=7)]) - if "mpileaks" in spec_string: + if "mpi" in spec: # It's a user, not a provider, so create the provider string # JCSDA fork only - no hashes in service_part # service_part = layout.spec["mpi"].format("{name}/{version}-{hash:7}") service_part = layout.spec["mpi"].format("{name}/{version}") + elif "python" in spec: + # It's a user, not a provider, so create the provider string + # service_part = layout.spec["python"].format("{name}/{version}-{hash:7}") + service_part = layout.spec["python"].format("{name}/{version}") + else: # Only relevant for providers, not users, of virtuals assert service_part in path_parts @@ -126,6 +140,16 @@ def test_compilers_provided_different_name( assert "compiler" in provides assert provides["compiler"] == spack.spec.Spec("intel-oneapi-compilers@=3.0") + @pytest.mark.parametrize("language", ["c", "cxx", "fortran"]) + def test_compiler_language_virtuals(self, factory, module_configuration, language): + """Tests all compiler virtuals for hierarchical module placement.""" + module_configuration("complex_hierarchy") + module, spec = factory(f"single-language-virtual +{language} %{language}=gcc@=10.2.1") + + requires = module.conf.requires + + assert "gcc@=10.2.1" in requires["compiler"] + def test_simple_case(self, modulefile_content, module_configuration): """Tests the generation of a simple Lua module file.""" @@ -315,16 +339,6 @@ def test_no_core_compilers(self, factory, module_configuration): with pytest.raises(spack.modules.lmod.CoreCompilersNotFoundError): module.write() - def test_non_virtual_in_hierarchy(self, factory, module_configuration): - """Ensures that if a non-virtual is in hierarchy, an exception will - be raised. - """ - module_configuration("non_virtual_in_hierarchy") - - module, spec = factory(mpileaks_spec_string) - with pytest.raises(spack.modules.lmod.NonVirtualInHierarchyError): - module.write() - def test_conflicts(self, modulefile_content, module_configuration): """Tests adding conflicts to the module.""" diff --git a/lib/spack/spack/test/modules/tcl.py b/lib/spack/spack/test/modules/tcl.py index 782a6a545737d8..3c1de68e17e087 100644 --- a/lib/spack/spack/test/modules/tcl.py +++ b/lib/spack/spack/test/modules/tcl.py @@ -42,11 +42,12 @@ def test_autoload_direct(self, modulefile_content, module_configuration): content = modulefile_content(mpileaks_spec_string) assert ( - len([x for x in content if "if {![info exists ::env(LMOD_VERSION_MAJOR)]} {" in x]) - == 1 + len([x for x in content if "if {![llength [info commands depends-on]]} {" in x]) == 1 ) - assert len([x for x in content if "depends-on " in x]) == 3 - assert len([x for x in content if "module load " in x]) == 3 + assert len([x for x in content if " proc depends-on {args} {" in x]) == 1 + assert len([x for x in content if " module load {*}$args" in x]) == 1 + # depends-on command defined once and used 3 times + assert len([x for x in content if "depends-on " in x]) == 4 # dtbuild1 has # - 1 ('run',) dependency @@ -56,11 +57,12 @@ def test_autoload_direct(self, modulefile_content, module_configuration): content = modulefile_content("dtbuild1") assert ( - len([x for x in content if "if {![info exists ::env(LMOD_VERSION_MAJOR)]} {" in x]) - == 1 + len([x for x in content if "if {![llength [info commands depends-on]]} {" in x]) == 1 ) - assert len([x for x in content if "depends-on " in x]) == 2 - assert len([x for x in content if "module load " in x]) == 2 + assert len([x for x in content if " proc depends-on {args} {" in x]) == 1 + assert len([x for x in content if " module load {*}$args" in x]) == 1 + # depends-on command defined once and used twice + assert len([x for x in content if "depends-on " in x]) == 3 # The configuration file sets the verbose keyword to False messages = [x for x in content if 'puts stderr "Autoloading' in x] @@ -73,11 +75,12 @@ def test_autoload_all(self, modulefile_content, module_configuration): content = modulefile_content(mpileaks_spec_string) assert ( - len([x for x in content if "if {![info exists ::env(LMOD_VERSION_MAJOR)]} {" in x]) - == 1 + len([x for x in content if "if {![llength [info commands depends-on]]} {" in x]) == 1 ) - assert len([x for x in content if "depends-on " in x]) == 6 - assert len([x for x in content if "module load " in x]) == 6 + assert len([x for x in content if " proc depends-on {args} {" in x]) == 1 + assert len([x for x in content if " module load {*}$args" in x]) == 1 + # depends-on command defined once and used 6 times + assert len([x for x in content if "depends-on " in x]) == 7 # dtbuild1 has # - 1 ('run',) dependency @@ -87,11 +90,12 @@ def test_autoload_all(self, modulefile_content, module_configuration): content = modulefile_content("dtbuild1") assert ( - len([x for x in content if "if {![info exists ::env(LMOD_VERSION_MAJOR)]} {" in x]) - == 1 + len([x for x in content if "if {![llength [info commands depends-on]]} {" in x]) == 1 ) - assert len([x for x in content if "depends-on " in x]) == 2 - assert len([x for x in content if "module load " in x]) == 2 + assert len([x for x in content if " proc depends-on {args} {" in x]) == 1 + assert len([x for x in content if " module load {*}$args" in x]) == 1 + # depends-on command defined once and used twice + assert len([x for x in content if "depends-on " in x]) == 3 def test_prerequisites_direct( self, modulefile_content, module_configuration, host_architecture_str @@ -132,7 +136,6 @@ def test_alter_environment(self, modulefile_content, module_configuration): assert len([x for x in content if "setenv FOO {foo}" in x]) == 0 assert len([x for x in content if "unsetenv BAR" in x]) == 0 assert len([x for x in content if "depends-on foo/bar" in x]) == 1 - assert len([x for x in content if "module load foo/bar" in x]) == 1 assert len([x for x in content if "setenv libdwarf_ROOT" in x]) == 1 def test_prepend_path_separator(self, modulefile_content, module_configuration): @@ -141,14 +144,14 @@ def test_prepend_path_separator(self, modulefile_content, module_configuration): module_configuration("module_path_separator") content = modulefile_content("module-path-separator") - assert len([x for x in content if "append-path COLON {foo}" in x]) == 1 - assert len([x for x in content if "prepend-path COLON {foo}" in x]) == 1 - assert len([x for x in content if "remove-path COLON {foo}" in x]) == 1 - assert len([x for x in content if "append-path --delim {;} SEMICOLON {bar}" in x]) == 1 - assert len([x for x in content if "prepend-path --delim {;} SEMICOLON {bar}" in x]) == 1 - assert len([x for x in content if "remove-path --delim {;} SEMICOLON {bar}" in x]) == 1 - assert len([x for x in content if "append-path --delim { } SPACE {qux}" in x]) == 1 - assert len([x for x in content if "remove-path --delim { } SPACE {qux}" in x]) == 1 + assert len([x for x in content if "append-path -d {:} COLON {foo}" in x]) == 1 + assert len([x for x in content if "prepend-path -d {:} COLON {foo}" in x]) == 1 + assert len([x for x in content if "remove-path -d {:} COLON {foo}" in x]) == 1 + assert len([x for x in content if "append-path -d {;} SEMICOLON {bar}" in x]) == 1 + assert len([x for x in content if "prepend-path -d {;} SEMICOLON {bar}" in x]) == 1 + assert len([x for x in content if "remove-path -d {;} SEMICOLON {bar}" in x]) == 1 + assert len([x for x in content if "append-path -d { } SPACE {qux}" in x]) == 1 + assert len([x for x in content if "remove-path -d { } SPACE {qux}" in x]) == 1 @pytest.mark.regression("11355") def test_manpath_setup(self, modulefile_content, module_configuration): @@ -162,13 +165,16 @@ def test_manpath_setup(self, modulefile_content, module_configuration): # manpath set by module with prepend-path content = modulefile_content("module-manpath-prepend") - assert len([x for x in content if "prepend-path MANPATH {/path/to/man}" in x]) == 1 - assert len([x for x in content if "prepend-path MANPATH {/path/to/share/man}" in x]) == 1 + assert len([x for x in content if "prepend-path -d {:} MANPATH {/path/to/man}" in x]) == 1 + assert ( + len([x for x in content if "prepend-path -d {:} MANPATH {/path/to/share/man}" in x]) + == 1 + ) assert len([x for x in content if "append-path MANPATH {}" in x]) == 1 # manpath set by module with append-path content = modulefile_content("module-manpath-append") - assert len([x for x in content if "append-path MANPATH {/path/to/man}" in x]) == 1 + assert len([x for x in content if "append-path -d {:} MANPATH {/path/to/man}" in x]) == 1 assert len([x for x in content if "append-path MANPATH {}" in x]) == 1 # manpath set by module with setenv @@ -237,14 +243,16 @@ def test_exclude(self, modulefile_content, module_configuration, host_architectu module_configuration("exclude") content = modulefile_content("mpileaks ^zmpi") - assert len([x for x in content if "module load " in x]) == 2 + # depends-on command defined once and used twice + assert len([x for x in content if "depends-on " in x]) == 3 with pytest.raises(FileNotFoundError): modulefile_content(f"callpath target={host_architecture_str}") content = modulefile_content(f"zmpi target={host_architecture_str}") - assert len([x for x in content if "module load " in x]) == 2 + # depends-on command defined once and used twice + assert len([x for x in content if "depends-on " in x]) == 3 def test_naming_scheme_compat(self, factory, module_configuration): """Tests backwards compatibility for naming_scheme key""" @@ -484,17 +492,17 @@ def test_autoload_with_constraints(self, modulefile_content, module_configuratio # Test the mpileaks that should have the autoloaded dependencies content = modulefile_content("mpileaks ^mpich2") - assert len([x for x in content if "depends-on " in x]) == 3 - assert len([x for x in content if "module load " in x]) == 3 + # depends-on command defined once and used 3 times + assert len([x for x in content if "depends-on " in x]) == 4 # Test the mpileaks that should NOT have the autoloaded dependencies content = modulefile_content("mpileaks ^mpich") assert ( - len([x for x in content if "if {![info exists ::env(LMOD_VERSION_MAJOR)]} {" in x]) - == 0 + len([x for x in content if "if {![llength [info commands depends-on]]} {" in x]) == 0 ) + assert len([x for x in content if " proc depends-on {args} {" in x]) == 0 + assert len([x for x in content if " module load {*}$args" in x]) == 0 assert len([x for x in content if "depends-on " in x]) == 0 - assert len([x for x in content if "module load " in x]) == 0 def test_modules_no_arch(self, factory, module_configuration): module_configuration("no_arch") diff --git a/lib/spack/spack/test/new_installer.py b/lib/spack/spack/test/new_installer.py index fa649b00e5d1c4..96a3ba867498a8 100644 --- a/lib/spack/spack/test/new_installer.py +++ b/lib/spack/spack/test/new_installer.py @@ -3,20 +3,31 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for the new_installer.py module""" -import pathlib as pathlb +import pathlib import sys +import time import pytest if sys.platform == "win32": pytest.skip("No Windows support", allow_module_level=True) -import spack.error -from spack.new_installer import OVERWRITE_GARBAGE_SUFFIX, PrefixPivoter +import spack.spec +from spack.new_installer import ( + OVERWRITE_GARBAGE_SUFFIX, + BinaryCacheMiss, + BuildGraph, + JobServer, + PackageInstaller, + PrefixPivoter, + _node_to_roots, + schedule_builds, +) +from spack.test.traverse import create_dag @pytest.fixture -def existing_prefix(tmp_path: pathlb.Path) -> pathlb.Path: +def existing_prefix(tmp_path: pathlib.Path) -> pathlib.Path: """Creates a standard existing prefix with content.""" prefix = tmp_path / "existing_prefix" prefix.mkdir() @@ -27,28 +38,22 @@ def existing_prefix(tmp_path: pathlb.Path) -> pathlb.Path: class TestPrefixPivoter: """Tests for the PrefixPivoter class.""" - def test_no_existing_prefix(self, tmp_path: pathlb.Path): + def test_no_existing_prefix(self, tmp_path: pathlib.Path): """Test installation when prefix doesn't exist yet.""" prefix = tmp_path / "new_prefix" - with PrefixPivoter(str(prefix), overwrite=False): + with PrefixPivoter(str(prefix)): prefix.mkdir() (prefix / "installed_file").write_text("content") assert prefix.exists() assert (prefix / "installed_file").read_text() == "content" - def test_existing_prefix_no_overwrite_raises(self, existing_prefix: pathlb.Path): - """Test that existing prefix raises error when overwrite=False.""" - with pytest.raises(spack.error.InstallError, match="already exists"): - with PrefixPivoter(str(existing_prefix), overwrite=False): - pass - - def test_overwrite_success_cleans_up_old_prefix( - self, tmp_path: pathlb.Path, existing_prefix: pathlb.Path + def test_existing_prefix_success_cleans_up_old_prefix( + self, tmp_path: pathlib.Path, existing_prefix: pathlib.Path ): - """Test that overwrite=True moves old prefix and cleans it up on success.""" - with PrefixPivoter(str(existing_prefix), overwrite=True): + """Test that an existing prefix is moved aside, and cleaned up on success.""" + with PrefixPivoter(str(existing_prefix)): assert not existing_prefix.exists() existing_prefix.mkdir() (existing_prefix / "new_file").write_text("new content") @@ -59,15 +64,12 @@ def test_overwrite_success_cleans_up_old_prefix( # Only the existing_prefix directory should remain assert len(list(tmp_path.iterdir())) == 1 - def test_overwrite_failure_restores_original_prefix( - self, tmp_path: pathlb.Path, existing_prefix: pathlb.Path + def test_existing_prefix_failure_restores_original_prefix( + self, tmp_path: pathlib.Path, existing_prefix: pathlib.Path ): - """Test that original prefix is restored when installation fails. - - Note: keep_prefix=True is passed but should be ignored since overwrite=True - takes precedence.""" + """Test that the original prefix is restored when installation fails.""" with pytest.raises(RuntimeError, match="simulated failure"): - with PrefixPivoter(str(existing_prefix), overwrite=True, keep_prefix=True): + with PrefixPivoter(str(existing_prefix), keep_prefix=False): existing_prefix.mkdir() (existing_prefix / "partial_file").write_text("partial") raise RuntimeError("simulated failure") @@ -75,22 +77,24 @@ def test_overwrite_failure_restores_original_prefix( assert existing_prefix.exists() assert (existing_prefix / "old_file").read_text() == "old content" assert not (existing_prefix / "partial_file").exists() - # Only the existing_prefix directory should remain + # Only the original prefix should remain assert len(list(tmp_path.iterdir())) == 1 - def test_overwrite_failure_no_partial_prefix_created(self, existing_prefix: pathlb.Path): - """Test restoration when failure occurs before any prefix is created.""" + def test_existing_prefix_failure_no_partial_prefix_created( + self, existing_prefix: pathlib.Path + ): + """Test restoration when failure occurs before the build creates the prefix dir.""" with pytest.raises(RuntimeError, match="early failure"): - with PrefixPivoter(str(existing_prefix), overwrite=True): + with PrefixPivoter(str(existing_prefix)): raise RuntimeError("early failure") assert existing_prefix.exists() assert (existing_prefix / "old_file").read_text() == "old content" - def test_overwrite_true_no_existing_prefix(self, tmp_path: pathlb.Path): - """Test that overwrite=True works fine when prefix doesn't exist.""" + def test_no_existing_prefix_success(self, tmp_path: pathlib.Path): + """Test that a fresh install with no pre-existing prefix works fine.""" prefix = tmp_path / "new_prefix" - with PrefixPivoter(str(prefix), overwrite=True): + with PrefixPivoter(str(prefix)): prefix.mkdir() (prefix / "installed_file").write_text("content") @@ -98,38 +102,81 @@ def test_overwrite_true_no_existing_prefix(self, tmp_path: pathlb.Path): # Only the new_prefix directory should remain assert len(list(tmp_path.iterdir())) == 1 - def test_keep_prefix_true_leaves_failed_install(self, tmp_path: pathlb.Path): - """Test that keep_prefix=True preserves the failed installation.""" + def test_keep_prefix_true_with_existing_prefix_keeps_failed_install( + self, tmp_path: pathlib.Path, existing_prefix: pathlib.Path + ): + """Test that keep_prefix=True keeps the failed install and discards the backup.""" + with pytest.raises(RuntimeError, match="simulated failure"): + with PrefixPivoter(str(existing_prefix), keep_prefix=True): + existing_prefix.mkdir() + (existing_prefix / "partial_file").write_text("partial content") + raise RuntimeError("simulated failure") + + # The failed prefix should be kept (not the original) + assert existing_prefix.exists() + assert (existing_prefix / "partial_file").exists() + assert not (existing_prefix / "old_file").exists() + # Backup should have been removed + assert len(list(tmp_path.iterdir())) == 1 + + def test_keep_prefix_false_removes_failed_install(self, tmp_path: pathlib.Path): + """Test that keep_prefix=False removes the failed installation (no pre-existing prefix).""" prefix = tmp_path / "new_prefix" with pytest.raises(RuntimeError, match="simulated failure"): - with PrefixPivoter(str(prefix), overwrite=False, keep_prefix=True): + with PrefixPivoter(str(prefix), keep_prefix=False): prefix.mkdir() (prefix / "partial_file").write_text("partial content") raise RuntimeError("simulated failure") - # Failed prefix should still exist + # Failed prefix should be removed + assert not prefix.exists() + # Nothing should remain + assert len(list(tmp_path.iterdir())) == 0 + + def test_keep_prefix_true_no_existing_prefix(self, tmp_path: pathlib.Path): + """Test failure with keep_prefix=True when no prefix existed beforehand.""" + prefix = tmp_path / "new_prefix" + + with pytest.raises(RuntimeError, match="simulated failure"): + with PrefixPivoter(str(prefix), keep_prefix=True): + prefix.mkdir() + (prefix / "partial_file").write_text("partial content") + raise RuntimeError("simulated failure") + + # The failed prefix should be kept assert prefix.exists() assert (prefix / "partial_file").exists() - assert (prefix / "partial_file").read_text() == "partial content" - # Only the failed prefix should remain + # No backup should exist assert len(list(tmp_path.iterdir())) == 1 - def test_keep_prefix_false_removes_failed_install(self, tmp_path: pathlb.Path): - """Test that keep_prefix=False removes the failed installation.""" + def test_failure_no_prefix_created(self, tmp_path: pathlib.Path): + """Test failure when the prefix directory was never created.""" prefix = tmp_path / "new_prefix" with pytest.raises(RuntimeError, match="simulated failure"): - with PrefixPivoter(str(prefix), overwrite=False, keep_prefix=False): - prefix.mkdir() - (prefix / "partial_file").write_text("partial content") + with PrefixPivoter(str(prefix), keep_prefix=False): + # Do NOT create the prefix directory raise RuntimeError("simulated failure") - # Failed prefix should be removed + # Prefix should not exist assert not prefix.exists() # Nothing should remain assert len(list(tmp_path.iterdir())) == 0 + def test_binary_cache_miss_with_keep_prefix_and_existing_prefix_restores_original( + self, tmp_path: pathlib.Path, existing_prefix: pathlib.Path + ): + """BinaryCacheMiss bypasses keep_prefix: original prefix is restored.""" + with pytest.raises(BinaryCacheMiss), PrefixPivoter(str(existing_prefix), keep_prefix=True): + existing_prefix.mkdir() + (existing_prefix / "partial_file").write_text("partial content") + raise BinaryCacheMiss("cache miss") + + assert (existing_prefix / "old_file").read_text() == "old content" + assert not (existing_prefix / "partial_file").exists() + assert len(list(tmp_path.iterdir())) == 1 + class FailingPrefixPivoter(PrefixPivoter): """Test subclass that can simulate filesystem failures.""" @@ -137,12 +184,11 @@ class FailingPrefixPivoter(PrefixPivoter): def __init__( self, prefix: str, - overwrite: bool, keep_prefix: bool = False, fail_on_restore: bool = False, fail_on_move_garbage: bool = False, ): - super().__init__(prefix, overwrite, keep_prefix) + super().__init__(prefix, keep_prefix) self.fail_on_restore = fail_on_restore self.fail_on_move_garbage = fail_on_move_garbage self.restore_rename_count = 0 @@ -167,10 +213,10 @@ class TestPrefixPivoterFailureRecovery: """Tests for edge cases and failure recovery in PrefixPivoter.""" def test_restore_failure_leaves_backup( - self, tmp_path: pathlb.Path, existing_prefix: pathlb.Path + self, tmp_path: pathlib.Path, existing_prefix: pathlib.Path ): """Test that if restoration fails, the backup is not deleted.""" - pivoter = FailingPrefixPivoter(str(existing_prefix), overwrite=True, fail_on_restore=True) + pivoter = FailingPrefixPivoter(str(existing_prefix), fail_on_restore=True) with pytest.raises(OSError, match="Simulated rename failure during restore"): with pivoter: @@ -183,12 +229,10 @@ def test_restore_failure_leaves_backup( assert len(list(tmp_path.iterdir())) == 2 def test_garbage_move_failure_leaves_backup( - self, tmp_path: pathlb.Path, existing_prefix: pathlb.Path + self, tmp_path: pathlib.Path, existing_prefix: pathlib.Path ): """Test that if moving the failed install to garbage fails, the backup is preserved.""" - pivoter = FailingPrefixPivoter( - str(existing_prefix), overwrite=True, fail_on_move_garbage=True - ) + pivoter = FailingPrefixPivoter(str(existing_prefix), fail_on_move_garbage=True) with pytest.raises(OSError, match="Simulated rename failure moving to garbage"): with pivoter: @@ -199,3 +243,472 @@ def test_garbage_move_failure_leaves_backup( assert (existing_prefix / "partial_file").exists() # Backup directory, failed prefix, and empty garbage directory should exist assert len(list(tmp_path.iterdir())) == 3 + + +class TestPackageInstallerConstructor: + """Tests for PackageInstaller constructor, especially capacity initialization.""" + + def test_capacity_explicit_concurrent_packages(self, temporary_store, mock_packages): + """Test that capacity is set correctly when concurrent_packages is explicitly provided.""" + spec = spack.spec.Spec("trivial-install-test-package") + spec._mark_concrete() + assert PackageInstaller([spec.package], concurrent_packages=5).capacity == 5 + assert PackageInstaller([spec.package], concurrent_packages=1).capacity == 1 + + def test_capacity_from_config_default_one( + self, temporary_store, mock_packages, mutable_config + ): + """Test that config value of 0 is treated as unlimited.""" + mutable_config.set("config:concurrent_packages", 0) + spec = spack.spec.Spec("trivial-install-test-package") + spec._mark_concrete() + assert PackageInstaller([spec.package]).capacity == sys.maxsize + + def test_capacity_from_config_non_zero(self, temporary_store, mock_packages, mutable_config): + """Test that non-0 config values are used as-is.""" + mutable_config.set("config:concurrent_packages", 1) + spec = spack.spec.Spec("trivial-install-test-package") + spec._mark_concrete() + assert PackageInstaller([spec.package]).capacity == 1 + + def test_no_binary_mirrors_forces_source_only( + self, temporary_store, mock_packages, mutable_config + ): + """With no binary mirrors configured, auto is overridden to source_only.""" + spec = spack.spec.Spec("trivial-install-test-package") + spec._mark_concrete() + installer = PackageInstaller([spec.package], root_policy="auto") + assert installer.root_policy == "source_only" + assert installer.dependencies_policy == "source_only" + + def test_no_binary_mirrors_preserves_cache_only( + self, temporary_store, mock_packages, mutable_config + ): + """Without binary mirrors, an explicit cache_only shouldn't turn into source_only.""" + spec = spack.spec.Spec("trivial-install-test-package") + spec._mark_concrete() + installer = PackageInstaller( + [spec.package], root_policy="cache_only", dependencies_policy="cache_only" + ) + assert installer.root_policy == "cache_only" + assert installer.dependencies_policy == "cache_only" + + +class _FakeBuildGraph: + """Minimal stand-in for BuildGraph in schedule_builds unit tests. + + Provides the two interface points that schedule_builds calls: + - .nodes (dict: dag_hash -> Spec) + - .enqueue_parents(dag_hash, pending_builds) + """ + + def __init__(self, specs): + self.nodes = {spec.dag_hash(): spec for spec in specs} + + def enqueue_parents(self, dag_hash, pending_builds): + """Remove dag_hash from nodes; no parents in these simple unit tests.""" + self.nodes.pop(dag_hash, None) + + +class TestScheduleBuilds: + """Unit tests for the module-level schedule_builds() function.""" + + def _make_spec(self, name): + """Return a minimal concrete spec suitable for locking and DB queries.""" + spec = spack.spec.Spec(name) + spec._mark_concrete() + return spec + + def _mark_installed(self, spec, store): + """Create the install directory structure and register the spec in the DB as installed.""" + store.layout.create_install_directory(spec) + store.db.add(spec, explicit=True) + + def test_not_installed_no_running_starts_build(self, temporary_store, mock_packages): + """A fresh spec with no running builds is added to to_start.""" + spec = self._make_spec("trivial-install-test-package") + pending = [spec.dag_hash()] + bg = _FakeBuildGraph([spec]) + jobserver = JobServer(num_jobs=2) + try: + result = schedule_builds( + pending, + bg, + temporary_store.db, + temporary_store.prefix_locker, + overwrite=set(), + overwrite_time=0.0, + capacity=1, + needs_jobserver_token=False, + jobserver=jobserver, + explicit=set(), + ) + assert not result.blocked + assert len(result.to_start) == 1 + assert result.to_start[0][0] == spec.dag_hash() + assert not result.newly_installed + assert not pending # removed from the pending list + finally: + for _, lock in result.to_start: + lock.release_write() + jobserver.close() + + def test_already_installed_yields_newly_installed(self, temporary_store, mock_packages): + """A spec already in the DB is returned in newly_installed, not in to_start.""" + spec = self._make_spec("trivial-install-test-package") + self._mark_installed(spec, temporary_store) + pending = [spec.dag_hash()] + bg = _FakeBuildGraph([spec]) + jobserver = JobServer(num_jobs=2) + try: + result = schedule_builds( + pending, + bg, + temporary_store.db, + temporary_store.prefix_locker, + overwrite=set(), + overwrite_time=0.0, + capacity=1, + needs_jobserver_token=False, + jobserver=jobserver, + explicit=set(), + ) + assert not result.blocked + assert not result.to_start + assert len(result.newly_installed) == 1 + assert result.newly_installed[0][0] == spec.dag_hash() + assert not pending # removed from the pending list + finally: + for _, _, lock in result.newly_installed: + lock.release_read() + jobserver.close() + + def test_no_jobserver_token_returns_empty(self, temporary_store, mock_packages): + """When has_running_builds=True and no token is available, nothing is started.""" + spec = self._make_spec("trivial-install-test-package") + pending = [spec.dag_hash()] + bg = _FakeBuildGraph([spec]) + # num_jobs=1 writes 0 tokens to the FIFO. Only the implicit token exists. + jobserver = JobServer(num_jobs=1) + try: + result = schedule_builds( + pending, + bg, + temporary_store.db, + temporary_store.prefix_locker, + overwrite=set(), + overwrite_time=0.0, + capacity=2, + needs_jobserver_token=True, + jobserver=jobserver, + explicit=set(), + ) + assert not result.blocked + assert not result.to_start + assert not result.newly_installed + assert len(pending) == 1 + finally: + jobserver.close() + + def test_all_locked_returns_blocked(self, temporary_store, mock_packages, monkeypatch): + """When all pending specs are locked externally, blocked_on_locks is True.""" + spec = self._make_spec("trivial-install-test-package") + pending = [spec.dag_hash()] + bg = _FakeBuildGraph([spec]) + jobserver = JobServer(num_jobs=2) + # Pre-register the lock in the prefix_locker cache, then patch try_acquire to fail. + lock = temporary_store.prefix_locker.lock(spec) + monkeypatch.setattr(lock, "try_acquire_write", lambda: False) + monkeypatch.setattr(lock, "try_acquire_read", lambda: False) + try: + result = schedule_builds( + pending, + bg, + temporary_store.db, + temporary_store.prefix_locker, + overwrite=set(), + overwrite_time=0.0, + capacity=2, + needs_jobserver_token=False, + jobserver=jobserver, + explicit=set(), + ) + assert result.blocked + assert not result.to_start + assert not result.newly_installed + assert len(pending) == 1 + finally: + jobserver.close() + + def test_overwrite_installed_spec_is_started(self, temporary_store, mock_packages): + """A spec in the overwrite set is scheduled even when already installed.""" + spec = self._make_spec("trivial-install-test-package") + self._mark_installed(spec, temporary_store) + pending = [spec.dag_hash()] + bg = _FakeBuildGraph([spec]) + jobserver = JobServer(num_jobs=2) + try: + result = schedule_builds( + pending, + bg, + temporary_store.db, + temporary_store.prefix_locker, + overwrite={spec.dag_hash()}, + overwrite_time=time.time() + 100, + capacity=1, + needs_jobserver_token=False, + jobserver=jobserver, + explicit=set(), + ) + assert not result.blocked + assert len(result.to_start) == 1 + assert result.to_start[0][0] == spec.dag_hash() + assert not result.newly_installed + finally: + for _, lock in result.to_start: + lock.release_write() + jobserver.close() + + def test_mixed_locked_unlocked(self, temporary_store, mock_packages, monkeypatch): + """Only the unlocked spec enters to_start when one spec is externally locked.""" + spec_a = self._make_spec("trivial-install-test-package") + spec_b = self._make_spec("trivial-smoke-test") + pending = [spec_a.dag_hash(), spec_b.dag_hash()] + bg = _FakeBuildGraph([spec_a, spec_b]) + jobserver = JobServer(num_jobs=4) + # Patch spec_a's lock to always fail, simulating an external write lock. + lock_a = temporary_store.prefix_locker.lock(spec_a) + monkeypatch.setattr(lock_a, "try_acquire_write", lambda: False) + monkeypatch.setattr(lock_a, "try_acquire_read", lambda: False) + try: + result = schedule_builds( + pending, + bg, + temporary_store.db, + temporary_store.prefix_locker, + overwrite=set(), + overwrite_time=0.0, + capacity=2, + needs_jobserver_token=False, + jobserver=jobserver, + explicit=set(), + ) + assert not result.blocked # spec_b was schedulable + started_hashes = {h for h, _ in result.to_start} + assert spec_b.dag_hash() in started_hashes + assert spec_a.dag_hash() not in started_hashes + assert not result.newly_installed + finally: + for _, lock in result.to_start: + lock.release_write() + jobserver.close() + + def test_write_locked_read_locked_installed_yields_newly_installed( + self, temporary_store, mock_packages, monkeypatch + ): + """Write lock fails but read lock succeeds and spec is installed: treated as done. + + Simulates the case where another process finished building and downgraded its write lock + to a read lock. The spec should appear in newly_installed. blocked remains True because no + write lock was obtained, preventing the jobserver from firing unnecessarily. + """ + spec = self._make_spec("trivial-install-test-package") + self._mark_installed(spec, temporary_store) + pending = [spec.dag_hash()] + bg = _FakeBuildGraph([spec]) + jobserver = JobServer(num_jobs=2) + lock = temporary_store.prefix_locker.lock(spec) + monkeypatch.setattr(lock, "try_acquire_write", lambda: False) + try: + result = schedule_builds( + pending, + bg, + temporary_store.db, + temporary_store.prefix_locker, + overwrite=set(), + overwrite_time=0.0, + capacity=2, + needs_jobserver_token=False, + jobserver=jobserver, + explicit=set(), + ) + assert result.blocked # no write lock was obtained; jobserver should not fire + assert not result.to_start + assert len(result.newly_installed) == 1 + dag_hash, installed_spec, lock = result.newly_installed[0] + assert dag_hash == spec.dag_hash() + assert installed_spec == spec + assert not pending # spec was removed from pending + finally: + for _, _, lock in result.newly_installed: + lock.release_read() + jobserver.close() + + def test_write_locked_read_locked_not_installed_still_blocked( + self, temporary_store, mock_packages, monkeypatch + ): + """Write lock fails, read lock succeeds, but spec is not in DB: retry later. + + Simulates the case where a concurrent process was killed mid-build. The read lock is + released and the spec stays in pending; blocked should remain True. + """ + spec = self._make_spec("trivial-install-test-package") + pending = [spec.dag_hash()] + bg = _FakeBuildGraph([spec]) + jobserver = JobServer(num_jobs=2) + lock = temporary_store.prefix_locker.lock(spec) + monkeypatch.setattr(lock, "try_acquire_write", lambda: False) + try: + result = schedule_builds( + pending, + bg, + temporary_store.db, + temporary_store.prefix_locker, + overwrite=set(), + overwrite_time=0.0, + capacity=2, + needs_jobserver_token=False, + jobserver=jobserver, + explicit=set(), + ) + assert result.blocked + assert not result.to_start + assert not result.newly_installed + assert pending == [spec.dag_hash()] # spec stays in pending for retry + finally: + jobserver.close() + + def test_overwrite_handled_by_concurrent_process(self, temporary_store, mock_packages): + """When a spec in overwrite was installed AFTER overwrite_time, another process did it.""" + spec = self._make_spec("trivial-install-test-package") + self._mark_installed(spec, temporary_store) # installation_time = now() + pending = [spec.dag_hash()] + bg = _FakeBuildGraph([spec]) + jobserver = JobServer(num_jobs=2) + try: + result = schedule_builds( + pending, + bg, + temporary_store.db, + temporary_store.prefix_locker, + overwrite={spec.dag_hash()}, + overwrite_time=0.0, # earlier than now() + capacity=1, + needs_jobserver_token=False, + jobserver=jobserver, + explicit=set(), + ) + assert not result.blocked + assert not result.to_start + assert len(result.newly_installed) == 1 + assert result.newly_installed[0][0] == spec.dag_hash() + finally: + for _, _, lock in result.newly_installed: + lock.release_read() + jobserver.close() + + def test_installed_implicit_explicit_set_produces_db_update( + self, temporary_store, mock_packages + ): + """An installed-implicit spec in explicit set produces a DbUpdate.""" + spec = self._make_spec("trivial-install-test-package") + temporary_store.layout.create_install_directory(spec) + temporary_store.db.add(spec, explicit=False) + pending = [spec.dag_hash()] + bg = _FakeBuildGraph([spec]) + jobserver = JobServer(num_jobs=2) + try: + result = schedule_builds( + pending, + bg, + temporary_store.db, + temporary_store.prefix_locker, + overwrite=set(), + overwrite_time=0.0, + capacity=1, + needs_jobserver_token=False, + jobserver=jobserver, + explicit={spec.dag_hash()}, + ) + assert len(result.to_mark_explicit) == 1 + assert result.to_mark_explicit[0].spec is spec + assert len(result.newly_installed) == 1 + finally: + for _, _, lock in result.newly_installed: + lock.release_read() + jobserver.close() + + +def test_nodes_to_roots(): + """Independent roots don't reach each other's exclusive nodes.""" + # A - B and C - D are disconnected graphs, A, B and C are "roots". + specs = create_dag(nodes=["A", "B", "C", "D"], edges=[("A", "B", "all"), ("C", "D", "all")]) + a, b, c, d = specs["A"], specs["B"], specs["C"], specs["D"] + node_to_roots = _node_to_roots([a, b, c]) + assert node_to_roots[a.dag_hash()] == frozenset([a.dag_hash()]) + assert node_to_roots[b.dag_hash()] == frozenset([a.dag_hash(), b.dag_hash()]) + assert node_to_roots[c.dag_hash()] == frozenset([c.dag_hash()]) + assert node_to_roots[d.dag_hash()] == frozenset([c.dag_hash()]) + + +def test_nodes_to_roots_shared_dependency(): + """A dependency shared by two roots is attributed to both.""" + specs = create_dag(nodes=["A", "B", "C"], edges=[("A", "C", "all"), ("B", "C", "all")]) + a, b, c = specs["A"], specs["B"], specs["C"] + node_to_roots = _node_to_roots([a, b]) + assert node_to_roots[a.dag_hash()] == frozenset([a.dag_hash()]) + assert node_to_roots[b.dag_hash()] == frozenset([b.dag_hash()]) + assert node_to_roots[c.dag_hash()] == frozenset([a.dag_hash(), b.dag_hash()]) + + +def test_expand_build_deps_source_only_includes_nested_build_deps(temporary_store): + """When dependencies_policy is source_only, expand_build_deps must include BUILD deps of + dynamically added specs, not just LINK|RUN. Otherwise those specs attempt to build from source + without their build tools in the graph.""" + # root --[build]--> build_tool --[build]--> nested_build_tool + # --[link]--> lib_dep + specs = create_dag( + nodes=["root", "build_tool", "nested_build_tool", "lib_dep"], + edges=[ + ("root", "build_tool", "build"), + ("build_tool", "nested_build_tool", "build"), + ("build_tool", "lib_dep", "link"), + ], + ) + root = specs["root"] + for s in specs.values(): + s._mark_concrete() + + # Construct a BuildGraph with root_policy="auto" so root's build deps are deferred. + bg = BuildGraph( + specs=[root], + root_policy="auto", + dependencies_policy="source_only", + include_build_deps=False, + install_package=True, + install_deps=True, + database=temporary_store.db, + ) + + # The initial graph should contain only root (build deps deferred for "auto" policy). + assert root.dag_hash() in bg.nodes + assert specs["build_tool"].dag_hash() not in bg.nodes + + # Simulate a cache miss: expand build deps for root. + pending = [] + with temporary_store.db.read_transaction(): + newly_added = bg.expand_build_deps( + [root.dag_hash()], pending, temporary_store.db, dependencies_policy="source_only" + ) + + added_hashes = set(newly_added) + + # build_tool must be added (direct BUILD dep of root) + assert specs["build_tool"].dag_hash() in added_hashes + + # lib_dep must be added (LINK dep of build_tool) + assert specs["lib_dep"].dag_hash() in added_hashes + + # nested_build_tool must also be added (BUILD dep of build_tool). This is the bug: without the + # fix, expand_build_deps only traverses LINK|RUN, so nested_build_tool is missing. + assert specs["nested_build_tool"].dag_hash() in added_hashes diff --git a/lib/spack/spack/test/oci/image.py b/lib/spack/spack/test/oci/image.py index bf724e20a402e4..ba13c75dab5dac 100644 --- a/lib/spack/spack/test/oci/image.py +++ b/lib/spack/spack/test/oci/image.py @@ -11,13 +11,13 @@ "image_ref, expected", [ ( - f"example.com:1234/a/b/c:tag@sha256:{'a'*64}", + f"example.com:1234/a/b/c:tag@sha256:{'a' * 64}", ("example.com:1234", "a/b/c", "tag", Digest.from_sha256("a" * 64)), ), ("example.com:1234/a/b/c:tag", ("example.com:1234", "a/b/c", "tag", None)), ("example.com:1234/a/b/c", ("example.com:1234", "a/b/c", "latest", None)), ( - f"example.com:1234/a/b/c@sha256:{'a'*64}", + f"example.com:1234/a/b/c@sha256:{'a' * 64}", ("example.com:1234", "a/b/c", "latest", Digest.from_sha256("a" * 64)), ), # ipv4 @@ -45,17 +45,17 @@ def test_name_parsing(image_ref, expected): "image_ref", [ # wrong order of tag and sha - f"example.com:1234/a/b/c@sha256:{'a'*64}:tag", + f"example.com:1234/a/b/c@sha256:{'a' * 64}:tag", # double tag "example.com:1234/a/b/c:tag:tag", # empty tag "example.com:1234/a/b/c:", # empty digest "example.com:1234/a/b/c@sha256:", - # unsupport digest algorithm - f"example.com:1234/a/b/c@sha512:{'a'*128}", + # unsupported digest algorithm + f"example.com:1234/a/b/c@sha512:{'a' * 128}", # invalid digest length - f"example.com:1234/a/b/c@sha256:{'a'*63}", + f"example.com:1234/a/b/c@sha256:{'a' * 63}", # whitespace "example.com:1234/a/b/c :tag", "example.com:1234/a/b/c: tag", diff --git a/lib/spack/spack/test/oci/integration_test.py b/lib/spack/spack/test/oci/integration_test.py index a6e74ee4d87078..770314f46f608a 100644 --- a/lib/spack/spack/test/oci/integration_test.py +++ b/lib/spack/spack/test/oci/integration_test.py @@ -256,7 +256,7 @@ def test_uploading_with_base_image_in_docker_image_manifest_v2_format( "history": [ { "created": "2015-10-31T22:22:54.690851953Z", - "created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /", + "created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /", # noqa: E501 } ], } diff --git a/lib/spack/spack/test/oci/mock_registry.py b/lib/spack/spack/test/oci/mock_registry.py index aa98ab07771e9b..807c6a7713b6b1 100644 --- a/lib/spack/spack/test/oci/mock_registry.py +++ b/lib/spack/spack/test/oci/mock_registry.py @@ -198,10 +198,9 @@ def put_manifest(self, req: Request, name: str, ref: str): else: for manifest in index_or_manifest["manifests"]: - assert ( - name, - manifest["digest"], - ) in self.manifests, "Missing manifest while uploading index" + assert (name, manifest["digest"]) in self.manifests, ( + "Missing manifest while uploading index" + ) self.manifests[(name, ref)] = index_or_manifest diff --git a/lib/spack/spack/test/oci/urlopen.py b/lib/spack/spack/test/oci/urlopen.py index 9c340a78bc148d..28249c7e617a2c 100644 --- a/lib/spack/spack/test/oci/urlopen.py +++ b/lib/spack/spack/test/oci/urlopen.py @@ -365,8 +365,8 @@ def test_registry_with_short_lived_bearer_tokens(): ("GET", "/v2/"), # 2: retry with bearer token ("GET", "/v2/"), # 3: with incorrect bearer token ("GET", "/v2/"), # 4: retry with new bearer token - ("GET", "/v2/"), # 5: with recyled correct bearer token - ("GET", "/v2/"), # 6: with recyled correct bearer token + ("GET", "/v2/"), # 5: with recycled correct bearer token + ("GET", "/v2/"), # 6: with recycled correct bearer token ] diff --git a/lib/spack/spack/test/package_class.py b/lib/spack/spack/test/package_class.py index ccc3f7df047596..c66ada17e43f0f 100644 --- a/lib/spack/spack/test/package_class.py +++ b/lib/spack/spack/test/package_class.py @@ -52,6 +52,9 @@ def mpileaks_possible_deps(mock_packages, mpi_names, compiler_names): "mpileaks": set(["callpath"] + mpi_names + compiler_names), "multi-provider-mpi": set(), "zmpi": set(["fake"] + compiler_names), + "compiler-with-deps": set(["binutils-for-test", "zlib"] + compiler_names), + "binutils-for-test": set(["zlib"] + compiler_names), + "zlib": set(), } return possible @@ -85,6 +88,9 @@ def mpi_names(mock_inspector): "mpileaks", "gcc", "llvm", + "compiler-with-deps", + "binutils-for-test", + "zlib", "multi-provider-mpi", "callpath", "dyninst", diff --git a/lib/spack/spack/test/packaging.py b/lib/spack/spack/test/packaging.py index d38dda9ea80106..3cfae44d84ed33 100644 --- a/lib/spack/spack/test/packaging.py +++ b/lib/spack/spack/test/packaging.py @@ -5,6 +5,7 @@ """ This test checks the binary packaging infrastructure """ + import argparse import os import pathlib diff --git a/lib/spack/spack/test/patch.py b/lib/spack/spack/test/patch.py index 3a8b61b29b43a7..28318b0b742fec 100644 --- a/lib/spack/spack/test/patch.py +++ b/lib/spack/spack/test/patch.py @@ -174,6 +174,33 @@ def test_patch_in_spec(mock_packages, config): ) +def test_stale_patch_cache_falls_back_to_fresh(mock_packages, config): + """spec.patches returns correct patches even when the stale in-memory cache is wrong.""" + spec = spack.concretize.concretize_one("patch@=1.0") + pkg_cls = spack.repo.PATH.get_pkg_class("patch") + + # Inject a stale PatchCache: foo_sha256 points to a non-existent patch file + stale_cache = spack.patch.PatchCache(repository=spack.repo.PATH) + stale_cache.index = { + foo_sha256: { + pkg_cls.fullname: { + "owner": pkg_cls.fullname, + "relative_path": "stale_wrong.patch", + "level": 1, + "working_dir": ".", + "reverse": False, + } + } + } + spack.repo.PATH._patch_index = stale_cache + spack.repo.PATH._index_is_fresh = False + + patches = spec.patches + + assert len(patches) == 2 + assert {p.relative_path for p in patches} == {"foo.patch", "baz.patch"} + + def test_patch_mixed_versions_subset_constraint(mock_packages, config): """If we have a package with mixed x.y and x.y.z versions, make sure that a patch applied to a version range of x.y.z versions is not applied to @@ -365,7 +392,11 @@ def test_conditional_patched_dependencies(mock_packages, config): def check_multi_dependency_patch_specs( - libelf, libdwarf, fake, owner, package_dir # specs + libelf, + libdwarf, + fake, + owner, + package_dir, # specs ): # parent spec properties """Validate patches on dependencies of patch-several-dependencies.""" # basic patch on libelf diff --git a/lib/spack/spack/test/provider_index.py b/lib/spack/spack/test/provider_index.py index 40062fed041fed..a402f2c90746ac 100644 --- a/lib/spack/spack/test/provider_index.py +++ b/lib/spack/spack/test/provider_index.py @@ -17,6 +17,7 @@ mpi@:10.0: set([zmpi])}, 'stuff': {stuff: set([externalvirtual])}} """ + import io import spack.repo @@ -72,3 +73,15 @@ def test_copy(mock_packages): p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.PATH) q = p.copy() assert p == q + + +def test_remove_providers(mock_packages): + """Test removing providers from the index.""" + p = ProviderIndex(specs=["mpich"], repository=spack.repo.PATH) + # Check that mpich is a provider for mpi + providers = p.providers_for("mpi") + assert any(spec.name == "mpich" for spec in providers) + p.remove_providers({"mpich"}) + # After removal, mpich should no longer be a provider for mpi + providers = p.providers_for("mpi") + assert not any(spec.name == "mpich" for spec in providers) diff --git a/lib/spack/spack/test/relocate_text.py b/lib/spack/spack/test/relocate_text.py index 418c5c3c766215..f80fc24ee118ef 100644 --- a/lib/spack/spack/test/relocate_text.py +++ b/lib/spack/spack/test/relocate_text.py @@ -163,7 +163,7 @@ def replace_and_expect(prefix_map, before, after=None, suffix_safety_size=7): # Finally, make sure that the regex is not greedily finding the LAST null byte # it should find the first null byte in the window. In this test we put one null - # at a distance where we cant keep a long enough suffix, and one where we can, + # at a distance where we can't keep a long enough suffix, and one where we can, # so we should expect failure when the first null is used. error_msg = "Cannot replace {!r} with {!r} in the C-string {!r}.".format( b"pkg-abcdef", b"pkg-xyzabc", b"pkg-abcdef" diff --git a/lib/spack/spack/test/repo.py b/lib/spack/spack/test/repo.py index 06a90559af9a3f..0cba89b5167fb1 100644 --- a/lib/spack/spack/test/repo.py +++ b/lib/spack/spack/test/repo.py @@ -83,9 +83,9 @@ def test_repo_last_mtime(mock_packages): modified_after = "\n ".join( f"{path} ({mtime})" for mtime, path in mtime_with_package_py if mtime > repo_mtime ) - assert ( - max_mtime <= repo_mtime - ), f"the following files were modified while running tests:\n {modified_after}" + assert max_mtime <= repo_mtime, ( + f"the following files were modified while running tests:\n {modified_after}" + ) assert max_mtime == repo_mtime, f"last_mtime incorrect for {max_file}" diff --git a/lib/spack/spack/test/reporters.py b/lib/spack/spack/test/reporters.py index ae8e9e713d9008..a829f273baf369 100644 --- a/lib/spack/spack/test/reporters.py +++ b/lib/spack/spack/test/reporters.py @@ -41,9 +41,7 @@ def test_reporters_extract_basics(): ==> [2022-02-15-18:44:21.250165] test: {0}: {1} ==> [2022-02-15-18:44:21.250200] '{2}' {3}: {0} -""".format( - name, desc, fake_bin, status - ).splitlines() +""".format(name, desc, fake_bin, status).splitlines() parts = spack.reporters.extract.extract_test_parts("fake", outputs) assert len(parts) == 1 @@ -63,9 +61,7 @@ def test_reporters_extract_no_parts(capfd): ==> Testing package fake-1.0-abcdefg ==> [2022-02-11-17:14:38.875259] Installing {0} to {1} {2} -""".format( - fake_install_test_root, fake_test_cache, status - ).splitlines() +""".format(fake_install_test_root, fake_test_cache, status).splitlines() parts = spack.reporters.extract.extract_test_parts("fake", outputs) err = capfd.readouterr()[1] @@ -125,9 +121,7 @@ def test_reporters_extract_skipped(state): outputs = """ ==> Testing package fake-1.0-abcdefg {0} -""".format( - expected - ).splitlines() +""".format(expected).splitlines() parts = spack.reporters.extract.extract_test_parts("fake", outputs) diff --git a/lib/spack/spack/test/sandbox.py b/lib/spack/spack/test/sandbox.py new file mode 100644 index 00000000000000..9c9e901848e83b --- /dev/null +++ b/lib/spack/spack/test/sandbox.py @@ -0,0 +1,205 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Unit tests for Linux Landlock sandboxing in the new installer.""" + +import sys + +import pytest + +if sys.platform != "linux": + pytest.skip("Landlock sandboxing is Linux only", allow_module_level=True) + +import os +import pathlib +import tempfile +from typing import List, Tuple + +import spack.concretize +import spack.sandbox +import spack.store +from spack.new_installer import _enable_sandbox + + +class SpyLandlockSandbox(spack.sandbox.LandlockSandbox): + """LandlockSandbox that records _syscall_* and _prctl_* calls.""" + + def __init__(self, abi_version: int = 3) -> None: + self._abi_version_override = abi_version + super().__init__() + self._fds: List[int] = [] + self.ruleset_fd = -1 + # (fs_flags, net_flags) + self.create_ruleset_calls: List[Tuple[int, int]] = [] + # (ruleset_fd, allowed_access, path_fd) + self.add_rule_calls: List[Tuple[int, int, int]] = [] + # (ruleset_fd, tsync_flag) + self.restrict_self_calls: List[Tuple[int, int]] = [] + self.prctl_called: bool = False + + def __del__(self): + for fd in self._fds: + os.close(fd) + + def _new_fd(self) -> int: + fd = os.open(os.devnull, os.O_RDONLY) + self._fds.append(fd) + return fd + + def _get_abi_version(self) -> int: + return self._abi_version_override + + def _syscall_create_ruleset(self, handled_access_fs: int, handled_access_net: int) -> int: + self.create_ruleset_calls.append((handled_access_fs, handled_access_net)) + self.ruleset_fd = self._new_fd() + return self.ruleset_fd + + def _syscall_add_rule(self, ruleset_fd: int, allowed_access: int, path_fd: int) -> None: + self.add_rule_calls.append((ruleset_fd, allowed_access, path_fd)) + + def _syscall_restrict_self(self, ruleset_fd: int, tsync_flag: int) -> None: + self.restrict_self_calls.append((ruleset_fd, tsync_flag)) + + def _prctl_no_new_privs(self) -> None: + self.prctl_called = True + + +def test_landlock_sandbox_syscall_args(tmp_path: pathlib.Path): + """Test that LandlockSandbox passes correct arguments to each syscall.""" + sandbox = SpyLandlockSandbox(abi_version=3) + + test_dir = tmp_path / "dir" + test_dir.mkdir() + test_file = test_dir / "file" + test_file.touch() + + sandbox.allow_read(test_dir) + sandbox.allow_write(test_file) + sandbox.apply(block_network=False) + + # Ruleset covers both read and write access; no network flags + [(fs_flags, net_flags)] = sandbox.create_ruleset_calls + assert fs_flags & spack.sandbox.FSAccess.READ_FILE + assert fs_flags & spack.sandbox.FSAccess.WRITE_FILE + assert net_flags == 0 + + # One rule per path, both using the same ruleset fd + assert len(sandbox.add_rule_calls) == 2 + for ruleset_fd, _access, path_fd in sandbox.add_rule_calls: + assert ruleset_fd == sandbox.ruleset_fd + assert path_fd > 0 + + # Read-only directory: has READ_DIR, no WRITE_FILE + dir_access = next( + a for _, a, _ in sandbox.add_rule_calls if a & spack.sandbox.FSAccess.READ_DIR + ) + assert not (dir_access & spack.sandbox.FSAccess.WRITE_FILE) + + # Write file: has WRITE_FILE, no READ_DIR (dir flags stripped for non-dirs) + file_access = next( + a for _, a, _ in sandbox.add_rule_calls if a & spack.sandbox.FSAccess.WRITE_FILE + ) + assert not (file_access & spack.sandbox.FSAccess.READ_DIR) + + # RESTRICT_SELF gets the correct ruleset fd + [(restrict_fd, tsync)] = sandbox.restrict_self_calls + assert restrict_fd == sandbox.ruleset_fd + assert tsync == 0 # ABI v3: no tsync flag + + assert sandbox.prctl_called + + +def test_landlock_sandbox_network_args(): + """Test that block_network=True sets the correct net flags in the ruleset.""" + sandbox = SpyLandlockSandbox(abi_version=4) + sandbox.apply(block_network=True) + + [(_, net_flags)] = sandbox.create_ruleset_calls + assert net_flags & spack.sandbox.LANDLOCK_ACCESS_NET_CONNECT_TCP + assert net_flags & spack.sandbox.LANDLOCK_ACCESS_NET_BIND_TCP + assert sandbox.prctl_called + + +class MockSandbox(spack.sandbox.Sandbox): + def __init__(self): + self.read_calls: List[Tuple[pathlib.Path, pathlib.Path]] = [] + self.write_calls: List[Tuple[pathlib.Path, pathlib.Path]] = [] + self.apply_calls: List[bool] = [] + + def _allow_read(self, original: pathlib.Path, resolved: pathlib.Path): + self.read_calls.append((original, resolved)) + + def _allow_write(self, original: pathlib.Path, resolved: pathlib.Path): + self.write_calls.append((original, resolved)) + + def apply(self, block_network=False): + self.apply_calls.append(block_network) + + +def test_enable_sandbox_paths( + monkeypatch, mock_packages, temporary_store: spack.store.Store, tmp_path: pathlib.Path +): + """Test that _enable_sandbox in new_installer calls allow_read/allow_write correctly.""" + mock_sandbox = MockSandbox() + monkeypatch.setattr(spack.sandbox, "get_sandbox", lambda: mock_sandbox) + + spec = spack.concretize.concretize_one("dependent-install") + + # Create prefix directories so resolved.exists() passes + pathlib.Path(spec.prefix).mkdir(parents=True, exist_ok=True) + for dep in spec.traverse(root=False): + pathlib.Path(dep.prefix).mkdir(parents=True, exist_ok=True) + + stage_path = tmp_path / "stage" + stage_path.mkdir() + + custom_write = tmp_path / "custom_write" + custom_write.mkdir() + + # Create a symlink to verify original vs resolved path logic + custom_read_target = tmp_path / "custom_read_target" + custom_read_target.mkdir() + custom_read_link = tmp_path / "custom_read_link" + custom_read_link.symlink_to(custom_read_target) + + # Ensure the sbang exists + temporary_store.install_sbang() + sbang_file = pathlib.Path(temporary_store.unpadded_root) / "bin" / "sbang" + + config = { + "enable": True, + "allow_read": [str(custom_read_link)], + "allow_write": [str(custom_write)], + "allow_network": True, + } + + _enable_sandbox(config, spec, str(stage_path)) + + allow_read_resolved = [c[1] for c in mock_sandbox.read_calls] + for dep in spec.traverse(root=False): + assert pathlib.Path(dep.prefix).resolve() in allow_read_resolved + + # Verify symlink resolution in read_calls + assert custom_read_target.resolve() in allow_read_resolved + assert (custom_read_link.absolute(), custom_read_target.resolve()) in mock_sandbox.read_calls + + # Verify sbang read + assert sbang_file.resolve() in allow_read_resolved + + allow_write_resolved = [c[1] for c in mock_sandbox.write_calls] + assert stage_path.resolve() in allow_write_resolved + assert pathlib.Path(spec.prefix).resolve() in allow_write_resolved + assert custom_write.resolve() in allow_write_resolved + assert pathlib.Path(tempfile.gettempdir()).resolve() in allow_write_resolved + + assert mock_sandbox.apply_calls == [False] + + +def test_sandbox_network_blocking_requires_abi_v4(): + """Test that blocking network access on an older kernel raises a RuntimeError.""" + sandbox = SpyLandlockSandbox(abi_version=3) + + with pytest.raises( + spack.sandbox.SandboxError, match="Blocking network access requires Landlock ABI v4\\+" + ): + sandbox.apply(block_network=True) diff --git a/lib/spack/spack/test/sbang.py b/lib/spack/spack/test/sbang.py index a220a1291d2db3..deda0bb38747da 100644 --- a/lib/spack/spack/test/sbang.py +++ b/lib/spack/spack/test/sbang.py @@ -5,6 +5,7 @@ """\ Test that Spack's shebang filtering works correctly. """ + import filecmp import os import pathlib @@ -111,7 +112,7 @@ def __init__(self, sbang_line): f.write(last_line) self.make_executable(self.luajit_shebang) - # Luajit occuring in text, not in shebang + # Luajit occurring in text, not in shebang self.luajit_textbang = os.path.join(self.tempdir, "luajit_in_text") with open(self.luajit_textbang, "w", encoding="utf-8") as f: f.write(short_line) @@ -126,7 +127,7 @@ def __init__(self, sbang_line): f.write(last_line) self.make_executable(self.node_shebang) - # Node occuring in text, not in shebang + # Node occurring in text, not in shebang self.node_textbang = os.path.join(self.tempdir, "node_in_text") with open(self.node_textbang, "w", encoding="utf-8") as f: f.write(short_line) @@ -141,7 +142,7 @@ def __init__(self, sbang_line): f.write(last_line) self.make_executable(self.php_shebang) - # php occuring in text, not in shebang + # php occurring in text, not in shebang self.php_textbang = os.path.join(self.tempdir, "php_in_text") with open(self.php_textbang, "w", encoding="utf-8") as f: f.write(short_line) @@ -280,9 +281,7 @@ def configure_group_perms(): read: world write: group group: {0} -""".format( - group_name - ) +""".format(group_name) ) spack.config.set("packages", conf, scope="user") @@ -334,7 +333,7 @@ def run_test_install_sbang(group): assert sbang_path.startswith(spack.store.STORE.unpadded_root) assert not os.path.exists(sbang_bin_dir) - sbang.install_sbang() + spack.store.STORE.install_sbang() check_sbang_installation(group) # put an invalid file in for sbang @@ -342,11 +341,11 @@ def run_test_install_sbang(group): with open(sbang_path, "w", encoding="utf-8") as f: f.write("foo") - sbang.install_sbang() + spack.store.STORE.install_sbang() check_sbang_installation(group) # install again and make sure sbang is still fine - sbang.install_sbang() + spack.store.STORE.install_sbang() check_sbang_installation(group) diff --git a/lib/spack/spack/test/schema.py b/lib/spack/spack/test/schema.py index 52675874125c7f..cb68cfdc0e07b5 100644 --- a/lib/spack/spack/test/schema.py +++ b/lib/spack/spack/test/schema.py @@ -10,6 +10,7 @@ from spack.vendor import jsonschema import spack.schema +import spack.schema.env import spack.util.spack_yaml as syaml from spack.llnl.util.lang import list_modules @@ -252,3 +253,8 @@ def test_spack_schemas_are_valid(): jsonschema.validate(module_schema, _draft_07_with_spack_extensions) except jsonschema.ValidationError as e: raise RuntimeError(f"Invalid JSON schema in {module_name}: {e.message}") from e + + +def test_env_schema_update_wrong_type(): + """Confirm passing the wrong type to env.update() results in no changes.""" + assert not spack.schema.env.update(["a/b"]) diff --git a/lib/spack/spack/test/spack_yaml.py b/lib/spack/spack/test/spack_yaml.py index ae04e0396127a3..20389626351cf6 100644 --- a/lib/spack/spack/test/spack_yaml.py +++ b/lib/spack/spack/test/spack_yaml.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Test Spack's custom YAML format.""" + import io import pathlib diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 4a283c55be029a..9d7b3250ad2761 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -4,6 +4,7 @@ """ These tests check Spec DAG operations using dummy packages. """ + import pytest import spack.concretize @@ -917,7 +918,7 @@ def test_query_dependents_edges(self, default_mock_concretization): edges_with_mpi = mpich.edges_from_dependents(virtuals=["mpi"]) assert edges_with_mpi == edges_of_link_type - # Check a node dependend upon by 2 parents + # Check a node depended upon by 2 parents assert len(mpileaks["libelf"].edges_from_dependents(depflag=dt.LINK)) == 2 diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 7046f1298db4cc..813c1ef1365146 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -12,6 +12,7 @@ import spack.llnl.util.lang import spack.package_base import spack.paths +import spack.repo import spack.solver.asp import spack.spec import spack.spec_parser @@ -444,7 +445,7 @@ def _propagated_flags(_spec): assert set(propagated_rhs) <= _propagated_flags(c2) def test_constrain_specs_by_hash(self, default_mock_concretization, database): - """Test that Specs specified only by their hashes can constrain eachother.""" + """Test that Specs specified only by their hashes can constrain each other.""" mpich_dag_hash = "/" + database.query_one("mpich").dag_hash() spec = Spec(mpich_dag_hash[:7]) assert spec.constrain(Spec(mpich_dag_hash)) is False @@ -622,6 +623,84 @@ def test_basic_satisfies_conditional_dep(self, default_mock_concretization): assert concrete.satisfies("^[when='^notapackage'] zmpi") assert not concrete.satisfies("^[when='^mpi'] zmpi") + def test_concrete_satisfies_does_not_consult_repo( + self, default_mock_concretization, monkeypatch + ): + """Tests that `satisfies()` on a concrete lhs doesn't need the provider index, when the rhs + contains a virtual name. + """ + concrete = default_mock_concretization("mpileaks ^mpich") + + # Reset the index, will raise if the `_provider_index` is ever removed as an attribute + monkeypatch.setattr(spack.repo.PATH, "_provider_index", None) + + # Basic match and mismatch cases. + assert concrete.satisfies("mpileaks") + assert not concrete.satisfies("zlib") + + # Virtuals on a direct edge + assert concrete.satisfies("%mpi") + assert concrete.satisfies("%mpi@3") + assert not concrete.satisfies("%mpi@5") + assert concrete.satisfies("%mpi=mpich") + assert not concrete.satisfies("%lapack") + + # Virtuals on a transitive edge + assert concrete.satisfies("^mpi") + assert concrete.satisfies("^mpi=mpich") + assert not concrete.satisfies("^lapack") + + # Concrete spec asking about one of its concrete deps. + mpich = concrete["mpich"] + assert mpich.satisfies("mpich") + assert mpich.satisfies("mpi") + + # We should not create again the index + assert spack.repo.PATH._provider_index is None + + def test_concrete_contains_does_not_consult_repo( + self, default_mock_concretization, monkeypatch + ): + """Tests that `foo in spec` on a concrete spec doesn't need the provider index, when the + item contains a virtual name. + """ + concrete = default_mock_concretization("mpileaks ^mpich") + + # Reset the index, will raise if the `_provider_index` is ever removed as an attribute + monkeypatch.setattr(spack.repo.PATH, "_provider_index", None) + + assert "mpi" in concrete + assert "c" in concrete + + # We should not create again the index + assert spack.repo.PATH._provider_index is None + + def test_abstract_satisfies_with_lhs_provider_rhs_virtual(self): + """If the left-hand side mentions a provider among dependencies and the right-hand side + mentions a virtual among its deps, we only have satisfaction if the edge attribute + specifies this virtual is provided.""" + assert not Spec("mpileaks ^mpich").satisfies("mpileaks ^mpi") + assert not Spec("mpileaks %mpich").satisfies("mpileaks %mpi") + assert Spec("mpileaks ^[virtuals=mpi] mpich").satisfies("mpileaks ^mpi") + assert Spec("mpileaks %[virtuals=mpi] mpich").satisfies("mpileaks ^mpi") + assert Spec("mpileaks %[virtuals=mpi] mpich").satisfies("mpileaks %mpi") + + def test_concrete_checks_on_virtual_names_dont_need_repo( + self, default_mock_concretization, monkeypatch + ): + """Tests that ``%mpi`` or similar on a concrete spec doesn't need the repo""" + concrete = default_mock_concretization("mpileaks ^mpich") + + # We don't need the repo + monkeypatch.setattr(spack.repo, "PATH", None) + + assert concrete.satisfies("%mpi") + assert concrete.satisfies("%c") + assert concrete.satisfies("%c=gcc") + assert concrete.satisfies("%mpi=mpich") + + assert not concrete.satisfies("%c,mpi=mpich") + def test_satisfies_single_valued_variant(self): """Tests that the case reported in https://github.com/spack/spack/pull/2386#issuecomment-282147639 @@ -1047,7 +1126,7 @@ class Pkg: fn = variant("foo", values=spack.variant.any_combination_of("fee", "foom"), default="bar") with pytest.raises(spack.directives.DirectiveError) as exc_info: fn(Pkg()) - assert " it is handled by an attribute of the 'values' " "argument" in str(exc_info.value) + assert " it is handled by an attribute of the 'values' argument" in str(exc_info.value) # We can't leave None as a default value fn = variant("foo", default=None) @@ -1169,7 +1248,7 @@ def test_splice_intransitive_complex(self, setup_complex_splice): assert spliced["pkg-e"]._build_spec is None # Because a copy of e is used, it does not have dependnets in the original specs assert set(spliced["pkg-e"].dependents()) == {spliced["pkg-b"], spliced["pkg-f"]} - # Build dependent edge to f because f originally dependended on the e this was copied from + # Build dependent edge to f because f originally depended on the e this was copied from assert set(spliced["pkg-e"].dependents(deptype=dt.BUILD)) == {spliced["pkg-b"]} assert spliced["pkg-f"].satisfies("pkg-f color=blue ^pkg-e color=red ^pkg-g@2 color=red") @@ -1978,6 +2057,21 @@ def test_constrain(factory, lhs_str, rhs_str, result, constrained_str): assert rhs == factory(constrained_str) +def test_constrain_dependencies_copies(mock_packages): + """Tests that constraining a spec with new deps makes proper copies, and does not accidentally + share dependency instances, leading to corruption of unrelated Spec instances.""" + x = Spec("root") + y = Spec("^foo") + z = Spec("%foo +bar") + assert x.constrain(y) + assert x == Spec("root ^foo") + assert x.constrain(z) + assert x == Spec("root %foo +bar") + assert not x.constrain(Spec("root %foo +bar")) # no new constraints + # now, double check that we did not mutate `y` after constraining `x` with `z`. + assert y == Spec("^foo") + + def test_abstract_hash_intersects_and_satisfies(default_mock_concretization): concrete: Spec = default_mock_concretization("pkg-a") hash = concrete.dag_hash() @@ -2041,7 +2135,9 @@ def test_virtual_queries_work_for_strings_and_lists(): """Ensure that ``dependencies()`` works with both virtuals=str and virtuals=[str, ...].""" parent, child = Spec("parent"), Spec("child") parent._add_dependency( - child, depflag=dt.BUILD, virtuals=("cxx", "fortran") # multi-char dep names + child, + depflag=dt.BUILD, + virtuals=("cxx", "fortran"), # multi-char dep names ) assert not parent.dependencies(virtuals="c") # not in virtuals but shares a char with cxx diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index f1e43742d2ea41..a9cebf91df8f78 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -12,7 +12,6 @@ import spack.binary_distribution import spack.cmd import spack.concretize -import spack.config import spack.error import spack.llnl.util.filesystem as fs import spack.platforms.test @@ -26,6 +25,7 @@ SpecParsingError, SpecTokenizationError, SpecTokens, + expand_toolchains, parse_one_or_raise, ) from spack.tokenize import Token @@ -417,12 +417,12 @@ def _specfile_for(spec_str, filename): ), # Version hash pair ( - rf"develop-branch-version@{'abc12'*8}=develop", + rf"develop-branch-version@{'abc12' * 8}=develop", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="develop-branch-version"), - Token(SpecTokens.VERSION_HASH_PAIR, value=f"@{'abc12'*8}=develop"), + Token(SpecTokens.VERSION_HASH_PAIR, value=f"@{'abc12' * 8}=develop"), ], - rf"develop-branch-version@{'abc12'*8}=develop", + rf"develop-branch-version@{'abc12' * 8}=develop", ), # Redundant specs ( @@ -1211,10 +1211,12 @@ def test_cli_spec_roundtrip(args, expected): ], ) def test_parse_toolchain(spec_str, toolchain, expected_roundtrip, mutable_config): - spack.config.CONFIG.set("toolchains", toolchain) + """Tests that toolchains are expanded correctly""" parser = SpecParser(spec_str) for expected in expected_roundtrip: - assert expected == str(parser.next_spec()) + result = parser.next_spec() + expand_toolchains(result, toolchain) + assert expected == str(result) @pytest.mark.parametrize( @@ -1653,7 +1655,7 @@ def test_parse_specfile_dependency(default_mock_concretization, tmp_path: pathli # Should also be accepted: "spack spec ..//libelf.yaml" spec = SpecParser( - f"libdwarf^..{os.path.sep}{specfile.parent.name}" f"{os.path.sep}{specfile.name}" + f"libdwarf^..{os.path.sep}{specfile.parent.name}{os.path.sep}{specfile.name}" ).next_spec() assert spec and spec["libelf"] == s["libelf"] diff --git a/lib/spack/spack/test/spec_yaml.py b/lib/spack/spack/test/spec_yaml.py index ade35eca9bea72..f6ac104dcab7dd 100644 --- a/lib/spack/spack/test/spec_yaml.py +++ b/lib/spack/spack/test/spec_yaml.py @@ -7,6 +7,7 @@ The YAML and JSON formats preserve DAG information in the spec. """ + import collections import collections.abc import gzip @@ -185,8 +186,8 @@ def test_ordered_read_not_required_for_consistent_dag_hash( # Dump to YAML and JSON yaml_string = syaml.dump(spec_dict, default_flow_style=False) yaml_string_rev = syaml.dump(spec_dict_rev, default_flow_style=False) - json_string = sjson.dump(spec_dict) - json_string_rev = sjson.dump(spec_dict_rev) + json_string = sjson.dumps(spec_dict) + json_string_rev = sjson.dumps(spec_dict_rev) # spec yaml is ordered like the spec dict assert yaml_string == spec_yaml @@ -440,9 +441,10 @@ def test_load_json_specfiles(specfile, expected_hash, reader_cls): assert s2.format("{compiler.name}") == "gcc" assert s2.format("{compiler.version}") != "none" - # Ensure satisfies still works with compilers + # Ensure satisfies works with compilers and direct dependencies assert s2.satisfies("%gcc") assert s2.satisfies("%gcc@9.4.0") + assert s2.satisfies("%zlib") def test_anchorify_1(): diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index aa817b4b53a097..eb2406e09bc7d5 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Test that the Stage class works correctly.""" + import collections import errno import getpass @@ -871,6 +872,21 @@ def test_develop_stage(self, develop_path, tmp_build_stage_dir): srctree2 = _create_tree_from_dir_recursive(srcdir) assert srctree2 == devtree + def test_develop_stage_without_reference_link(self, develop_path, tmp_build_stage_dir): + """Check that develop stages can be created without creating a reference link""" + devtree, srcdir = develop_path + stage = DevelopStage("test-stage", srcdir, reference_link=None) + stage.create() + srctree1 = _create_tree_from_dir_recursive(stage.source_path) + assert srctree1 == devtree + + stage.destroy() + # Make sure destroying the stage doesn't change anything + # about the path + assert not os.path.exists(stage.path) + srctree2 = _create_tree_from_dir_recursive(srcdir) + assert srctree2 == devtree + def test_stage_create_replace_path(tmp_build_stage_dir): """Ensure stage creation replaces a non-directory path.""" diff --git a/lib/spack/spack/test/tag.py b/lib/spack/spack/test/tag.py index 8b9d565cf29189..80903ab4e49149 100644 --- a/lib/spack/spack/test/tag.py +++ b/lib/spack/spack/test/tag.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for tag index cache files.""" + import io import pytest @@ -146,7 +147,6 @@ def test_tag_no_tags(mock_packages): def test_tag_update_package(mock_packages): mock_index = mock_packages.tag_index index = spack.tag.TagIndex() - for name in spack.repo.all_package_names(): - index.update_package(name, repo=mock_packages) + index.update_packages(set(spack.repo.all_package_names()), repo=mock_packages) ensure_tags_results_equal(mock_index.tags, index.tags) diff --git a/lib/spack/spack/test/test_suite.py b/lib/spack/spack/test/test_suite.py index 678a5db2fb3e87..c876428a0fd4ca 100644 --- a/lib/spack/spack/test/test_suite.py +++ b/lib/spack/spack/test/test_suite.py @@ -205,12 +205,6 @@ def test_test_function_names(mock_packages, install_mockery, virtuals, expected) assert sorted(tests) == sorted(expected) -def test_test_functions_fails(): - """Confirm test_functions raises error if no package.""" - with pytest.raises(ValueError, match="Expected a package"): - spack.install_test.test_functions(str) - - def test_test_functions_pkgless(mock_packages, install_mockery, ensure_debug, capfd): """Confirm works for package providing a package-less virtual.""" spec = spack.concretize.concretize_one("simple-standalone-test") diff --git a/lib/spack/spack/test/traverse.py b/lib/spack/spack/test/traverse.py index 4de1b0a32d0c48..b4b68ccc8f5b6c 100644 --- a/lib/spack/spack/test/traverse.py +++ b/lib/spack/spack/test/traverse.py @@ -237,7 +237,7 @@ def test_breadth_first_versus_depth_first_tree(abstract_specs_chain): for (depth, edge) in traverse.traverse_tree([s], cover="nodes", depth_first=False) ] == [(0, "chain-a"), (1, "chain-b"), (1, "chain-c"), (1, "chain-d")] - # DFS will disover all nodes along the chain a -> b -> c -> d. + # DFS will discover all nodes along the chain a -> b -> c -> d. assert [ (depth, edge.spec.name) for (depth, edge) in traverse.traverse_tree([s], cover="nodes", depth_first=True) @@ -405,7 +405,7 @@ def test_traverse_edges_topo(abstract_specs_toposort): for e in traverse.traverse_edges(input_specs, order="topo", cover="edges", root=False) ] - # See figure above, we have 7 edges (excluding artifical ones to the root) + # See figure above, we have 7 edges (excluding artificial ones to the root) assert set(edges) == set( [("A", "B"), ("A", "C"), ("B", "F"), ("B", "G"), ("C", "D"), ("D", "B"), ("E", "D")] ) @@ -435,7 +435,7 @@ def test_traverse_nodes_no_deps(abstract_specs_dtuse): def test_topo_is_bfs_for_trees(cover): """For trees, both DFS and BFS produce a topological order, but BFS is the most sensible for our applications, where we typically want to avoid that transitive dependencies shadow direct - depenencies in global search paths, etc. This test ensures that for trees, the default topo + dependencies in global search paths, etc. This test ensures that for trees, the default topo order coincides with BFS.""" binary_tree = create_dag( nodes=["A", "B", "C", "D", "E", "F", "G"], diff --git a/lib/spack/spack/test/util/editor.py b/lib/spack/spack/test/util/editor.py index ba4cdf6b98bcf4..2cfe5b409420ab 100644 --- a/lib/spack/spack/test/util/editor.py +++ b/lib/spack/spack/test/util/editor.py @@ -30,7 +30,7 @@ def clean_env_vars(): @pytest.fixture(autouse=True) def working_editor_test_env(working_env): - """Don't leak environent variables between functions here.""" + """Don't leak environment variables between functions here.""" # parameterized fixture for editor var names diff --git a/lib/spack/spack/test/util/environment.py b/lib/spack/spack/test/util/environment.py index 40f9f92bc643b1..9bba875d9d05e6 100644 --- a/lib/spack/spack/test/util/environment.py +++ b/lib/spack/spack/test/util/environment.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Test Spack's environment utility functions.""" + import os import pathlib import sys diff --git a/lib/spack/spack/test/util/executable.py b/lib/spack/spack/test/util/executable.py index afd666bec27113..b44e449c9ff312 100644 --- a/lib/spack/spack/test/util/executable.py +++ b/lib/spack/spack/test/util/executable.py @@ -32,9 +32,7 @@ def test_read_unicode(tmp_path: pathlib.Path, working_env): f.write( """#!{0} print(u'\\xc3') -""".format( - sys.executable - ) +""".format(sys.executable) ) # make it executable @@ -163,3 +161,17 @@ def test_construct_from_pathlib(mock_executable): path = mock_executable("hello", output=f"echo {expected}\n") hello = ex.Executable(path) assert expected in hello(output=str) + + +def test_exe_disallows_str_split_as_input(mock_executable): + path = mock_executable("hello", output="echo hi\n") + hello = ex.Executable(path) + with pytest.raises(ValueError): + hello(input=str.split) + + +def test_exe_disallows_callable_as_output(mock_executable): + path = mock_executable("hello", output="echo hi\n") + hello = ex.Executable(path) + with pytest.raises(ValueError): + hello(output=lambda line: line) diff --git a/lib/spack/spack/test/util/file_cache.py b/lib/spack/spack/test/util/file_cache.py index 47f4cd52961a12..25181f31dacc0c 100644 --- a/lib/spack/spack/test/util/file_cache.py +++ b/lib/spack/spack/test/util/file_cache.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Test Spack's FileCache.""" + import os import pathlib @@ -45,10 +46,10 @@ def test_failed_write_and_read_cache_file(file_cache): raise RuntimeError("foobar") # Cache dir should have exactly one (lock) file - assert os.listdir(file_cache.root) == [".test.yaml.lock"] + assert os.listdir(file_cache.root) == [".lock"] # File does not exist - assert not file_cache.init_entry("test.yaml") + assert not os.path.exists(file_cache.cache_path("test.yaml")) def test_write_and_remove_cache_file(file_cache): @@ -84,39 +85,37 @@ def test_write_and_remove_cache_file(file_cache): @pytest.mark.not_on_windows("Not supported on Windows (yet)") @pytest.mark.skipif(fs.getuid() == 0, reason="user is root") -def test_cache_init_entry_fails(file_cache): - """Test init_entry failures.""" +def test_bad_cache_permissions(file_cache, request): + """Test that transactions raise CacheError on permission problems.""" relpath = fs.join_path("test-dir", "read-only-file.txt") cachefile = file_cache.cache_path(relpath) fs.touchp(cachefile) - # Ensure directory causes exception + # A directory where a file is expected raises CacheError on read with pytest.raises(CacheError, match="not a file"): - file_cache.init_entry(os.path.dirname(relpath)) + with file_cache.read_transaction(os.path.dirname(relpath)) as _: + pass - # Ensure non-readable file causes exception + # A directory where a file is expected raises CacheError on write + with pytest.raises(CacheError, match="not a file"): + with file_cache.write_transaction(os.path.dirname(relpath)) as _: + pass + + # A non-readable file raises CacheError on read os.chmod(cachefile, 0o200) + request.addfinalizer(lambda c=cachefile: os.chmod(c, 0o600)) with pytest.raises(CacheError, match="Cannot access cache file"): - file_cache.init_entry(relpath) - - # Ensure read-only parent causes exception - relpath = fs.join_path("test-dir", "another-file.txxt") - cachefile = file_cache.cache_path(relpath) - os.chmod(os.path.dirname(cachefile), 0o400) - with pytest.raises(CacheError, match="Cannot access cache dir"): - file_cache.init_entry(relpath) - - -@pytest.mark.skipif(fs.getuid() == 0, reason="user is root") -def test_cache_write_readonly_cache_fails(file_cache): - """Test writing a read-only cached file.""" - filename = "read-only-file.txt" - path = file_cache.cache_path(filename) - fs.touch(path) - os.chmod(path, 0o400) - - with pytest.raises(CacheError, match="Insufficient permissions to write"): - file_cache.write_transaction(filename) + with file_cache.read_transaction(relpath) as _: + pass + + # A read-only parent directory raises CacheError on write + relpath2 = fs.join_path("test-dir", "another-file.txxt") + parent = str(file_cache.cache_path(relpath2).parent) + os.chmod(parent, 0o400) + request.addfinalizer(lambda p=parent: os.chmod(p, 0o700)) + with pytest.raises(CacheError): + with file_cache.write_transaction(relpath2) as _: + pass @pytest.mark.regression("31475") diff --git a/lib/spack/spack/test/util/log_parser.py b/lib/spack/spack/test/util/log_parser.py index be933b725e0fda..2eedd23a2684b0 100644 --- a/lib/spack/spack/test/util/log_parser.py +++ b/lib/spack/spack/test/util/log_parser.py @@ -2,9 +2,13 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import io import pathlib +import re -from spack.util.ctest_log_parser import CTestLogParser +from spack.llnl.util.tty.color import color_when +from spack.util.ctest_log_parser import CTestLogParser, LogEvent, _optimize_regexes +from spack.util.log_parse import make_log_context def test_log_parser(tmp_path: pathlib.Path): @@ -28,10 +32,197 @@ def test_log_parser(tmp_path: pathlib.Path): ) parser = CTestLogParser() - errors, warnings = parser.parse(str(log_file)) + errors, warnings, _ = parser.parse(str(log_file)) assert len(errors) == 4 assert all(e.text.endswith("E") for e in errors) assert len(warnings) == 1 assert all(w.text.endswith("W") for w in warnings) + + +def test_log_parser_stream(): + """parse() accepts a file-like object.""" + log = io.StringIO( + "error: weird_error.c:145: something weird happened E\n" + "checking for gcc... irrelevant line\n" + "/var/tmp/build/foo.py:60: warning: some weird warning W\n" + ) + parser = CTestLogParser() + errors, warnings, _ = parser.parse(log) + + assert len(errors) == 1 + assert errors[0].text.endswith("E") + assert len(warnings) == 1 + assert warnings[0].text.endswith("W") + + +def test_log_parser_preserves_leading_whitespace(): + """Leading whitespace (e.g. compiler caret underlines) must not be stripped.""" + log = io.StringIO( + "/path/to/file.c:10: error: use of undeclared identifier 'x'\n" + " int y = x + 1;\n" + " ^\n" + ) + parser = CTestLogParser() + errors, _, _ = parser.parse(log, context=6) + + assert len(errors) == 1 + assert errors[0].post_context[0] == " int y = x + 1;" + assert errors[0].post_context[1] == " ^" + + +def test_make_log_context_merges_overlapping_events(tmp_path: pathlib.Path): + """Overlapping or adjacent context windows should produce a single merged block.""" + + # Two errors close together: lines 5 and 10 with context=3 means windows overlap. + lines = [f"line {i}\n" for i in range(1, 21)] + lines[4] = "error: first problem\n" # line 5 + lines[9] = "error: second problem\n" # line 10 + + log_file = tmp_path / "log.txt" + log_file.write_text("".join(lines)) + + parser = CTestLogParser() + errors, warnings, _ = parser.parse(str(log_file), context=3) + + log_events = sorted([*errors, *warnings], key=lambda e: e.line_no) + output = make_log_context(log_events) + + # Should be exactly one header for the merged block, not two. + assert output.count("-- lines") == 1 + + # The header should cover the full merged range. + assert "-- lines 2 to 13 --" in output + + +def test_make_log_context_warning_in_error_context_keeps_yellow(tmp_path: pathlib.Path): + """A warning line inside an error's context window must be highlighted yellow, not red.""" + # Line 5 = error, line 8 = warning, context=3 so error window covers lines 2-11 + # meaning the warning at line 8 falls inside the error's context. + lines = [f"line {i}\n" for i in range(1, 16)] + lines[4] = "error: something broke\n" # line 5 + lines[7] = "/tmp/foo.c:1: warning: something fishy\n" # line 8 + + log_file = tmp_path / "log.txt" + log_file.write_text("".join(lines)) + + parser = CTestLogParser() + errors, warnings, _ = parser.parse(str(log_file), context=3) + + assert len(errors) == len(warnings) == 1 + + log_events = sorted([*errors, *warnings], key=lambda e: e.line_no) + + with color_when("always"): + output = make_log_context(log_events) + + # The error line should be red (ANSI 91), the warning yellow (ANSI 93). + assert "\x1b[0;91m> " in output and "something broke" in output + assert "\x1b[0;93m> " in output and "something fishy" in output + + +def test_log_parser_non_utf8_bytes(tmp_path: pathlib.Path): + """parse() does not raise UnicodeDecodeError on non-UTF-8 log files.""" + log_file = tmp_path / "log.bin" + log_file.write_bytes(b"checking things...\nerror: \x80\xff something broke\ndone\n") + parser = CTestLogParser() + errors, _, _ = parser.parse(str(log_file)) + assert len(errors) == 1 + + +def test_tail_renders_as_plain_context(): + """A LogEvent should render all lines as plain context with no highlighting.""" + lines = ["tail line 1", "tail line 2", "tail line 3"] + section = LogEvent(text=lines[-1], line_no=100, pre_context=lines[:-1]) + + with color_when(False): + output = make_log_context([section]) + + assert "-- lines 98 to 100 --" in output + # All lines should be plain context (indented with two spaces, no "> " prefix) + assert " tail line 1\n" in output + assert " tail line 2\n" in output + assert " tail line 3\n" in output + assert "> " not in output + + +def test_tail_overlapping_with_error(): + """Tail lines overlapping with an error's context should not be duplicated.""" + log = io.StringIO("line 1\nline 2\nline 3\nerror: something broke\nline 5\nline 6\nline 7\n") + parser = CTestLogParser() + errors, _, tail = parser.parse(log, context=2, tail=3) + assert len(errors) == 1 + assert tail is not None + + with color_when(False): + output = make_log_context([*errors, tail]) + + # "line 5" and "line 6" appear in both the error context and the tail, + # but should only appear once in the output + assert output.count("line 5") == 1 + assert output.count("line 6") == 1 + assert output.count("line 7") == 1 + + +def test_tail_only(): + """A LogEvent with no errors/warnings renders correctly.""" + lines = ["final line 1", "final line 2"] + section = LogEvent(text=lines[-1], line_no=51, pre_context=lines[:-1]) + + with color_when(False): + output = make_log_context([section]) + + assert "-- lines 50 to 51 --" in output + assert " final line 1\n" in output + assert " final line 2\n" in output + + +class TestOptimizeRegexes: + def test_groups_by_first_char(self): + """Regexes sharing a first character are combined into one.""" + result = _optimize_regexes(["bar", "far", "foo"]) + assert len(result) == 2 + assert result == ["bar", "far|foo"] + + def test_singletons_unchanged(self): + """A regex that is the only one with its prefix is kept as-is.""" + result = _optimize_regexes(["^unique pattern"]) + assert result == ["^unique pattern"] + + def test_escaping(self): + """Regexes starting with the same metacharacter are grouped too.""" + result = _optimize_regexes(["\\(foo\\)", "\\(bar\\)", "\\*", "[abc]"]) + assert len(result) == 3 + assert "\\(bar\\)|\\(foo\\)" in result + assert "\\*" in result + assert "[abc]" in result + + def test_semantics_preserved(self): + """Optimized regexes match the same strings as the originals.""" + originals = [ + "^FAIL: ", + "^FATAL: ", + "^failed ", + ": error", + ": warning", + "make: Fatal error", + "make\\[.*\\]: \\*\\*\\*", + ] + test_lines = [ + "FAIL: test_something", + "FATAL: crash", + "failed to build", + "foo.c: error: syntax", + "foo.c: warning: unused", + "make: Fatal error in target", + "make[1]: *** Error 1", + "this matches nothing", + ] + compiled_orig = [re.compile(r) for r in originals] + compiled_opt = [re.compile(r) for r in _optimize_regexes(originals)] + + for line in test_lines: + orig_match = any(r.search(line) for r in compiled_orig) + opt_match = any(r.search(line) for r in compiled_opt) + assert orig_match == opt_match, f"mismatch on {line!r}" diff --git a/lib/spack/spack/test/util/module_cmd.py b/lib/spack/spack/test/util/module_cmd.py new file mode 100644 index 00000000000000..d348a1bf4503d9 --- /dev/null +++ b/lib/spack/spack/test/util/module_cmd.py @@ -0,0 +1,57 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os + +import pytest + +import spack.util.module_cmd + + +@pytest.mark.not_on_windows("Module files are not supported on Windows") +def test_load_module_success(monkeypatch, working_env): + """Test that load_module properly handles successful module loads. + + This is a very lightweight test that only confirms that successful + loads are not flagged as failed.""" + + # Mock the module function to simulate a successful module load + def mock_module(*args, **kwargs): + if args[0] == "show": + return "" + elif args[0] == "load": + # Simulate successful module load by adding to LOADEDMODULES + current_modules = os.environ.get("LOADEDMODULES", "") + if current_modules: + os.environ["LOADEDMODULES"] = f"{current_modules}:{args[1]}" + else: + os.environ["LOADEDMODULES"] = args[1] + + monkeypatch.setattr(spack.util.module_cmd, "module", mock_module) + + # This should succeed + spack.util.module_cmd.load_module("test_module") + spack.util.module_cmd.load_module("test_module_2") + + # Confirm LOADEDMODULES was modified + assert "test_module:test_module_2" in os.environ["LOADEDMODULES"] + + +@pytest.mark.not_on_windows("Module files are not supported on Windows") +def test_load_module_failure(monkeypatch, working_env): + """Test that load_module raises an exception when a module load fails.""" + + # Mock the module function to simulate a failed module load + def mock_module(*args, **kwargs): + if args[0] == "show": + return "" + elif args[0] == "load": + # Simulate module load failure by not changing LOADEDMODULES + pass + + monkeypatch.setattr(spack.util.module_cmd, "module", mock_module) + + # This should fail with ModuleLoadError + with pytest.raises(spack.util.module_cmd.ModuleLoadError): + spack.util.module_cmd.load_module("non_existent_module") diff --git a/lib/spack/spack/test/util/path.py b/lib/spack/spack/test/util/path.py index 8f4ad0c7122a69..8907e0cad497d5 100644 --- a/lib/spack/spack/test/util/path.py +++ b/lib/spack/spack/test/util/path.py @@ -41,80 +41,93 @@ def test_sanitize_filename(): # This class pertains to path string padding manipulation specifically # which is used for binary caching. This functionality is not supported # on Windows as of yet. -@pytest.mark.not_on_windows("Padding funtionality unsupported on Windows") +@pytest.mark.not_on_windows("Padding functionality unsupported on Windows") +@pytest.mark.parametrize("as_bytes", [False, True]) class TestPathPadding: + @pytest.fixture(autouse=True) + def setup(self, as_bytes: bool): + #: The filter function, either for bytes or str + self.filter = sup.padding_filter_bytes if as_bytes else sup.padding_filter + #: A converter of str -> bytes if we're testing the bytes filter + self.convert = lambda s: s.encode("ascii") if as_bytes else s + @pytest.mark.parametrize("padded,fixed", zip(padded_lines, fixed_lines)) def test_padding_substitution(self, padded, fixed): """Ensure that all padded lines are unpadded correctly.""" - assert fixed == sup.padding_filter(padded) + assert self.convert(fixed) == self.filter(self.convert(padded)) def test_no_substitution(self): """Ensure that a line not containing one full path placeholder is not modified.""" partial = "--prefix=/Users/gamblin2/padding-log-test/opt/__spack_path_pla/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga'" # noqa: E501 - assert sup.padding_filter(partial) == partial + p = self.convert(partial) + assert self.filter(p) is p # Test fast-path identity def test_short_substitution(self): """Ensure that a single placeholder path component is replaced""" short = "--prefix=/Users/gamblin2/padding-log-test/opt/__spack_path_placeholder__/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga'" # noqa: E501 short_subst = "--prefix=/Users/gamblin2/padding-log-test/opt/[padded-to-63-chars]/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga'" # noqa: E501 - assert short_subst == sup.padding_filter(short) + assert self.convert(short_subst) == self.filter(self.convert(short)) def test_partial_substitution(self): """Ensure that a single placeholder path component is replaced""" short = "--prefix=/Users/gamblin2/padding-log-test/opt/__spack_path_placeholder__/__spack_p/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga'" # noqa: E501 short_subst = "--prefix=/Users/gamblin2/padding-log-test/opt/[padded-to-73-chars]/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga'" # noqa: E501 - assert short_subst == sup.padding_filter(short) + assert self.convert(short_subst) == self.filter(self.convert(short)) def test_longest_prefix_re(self): """Test that longest_prefix_re generates correct regular expressions.""" assert "(s(?:t(?:r(?:i(?:ng?)?)?)?)?)" == sup.longest_prefix_re("string", capture=True) assert "(?:s(?:t(?:r(?:i(?:ng?)?)?)?)?)" == sup.longest_prefix_re("string", capture=False) - def test_output_filtering(self, capfd, install_mockery, mutable_config): - """Test filtering padding out of tty messages.""" - long_path = "/" + "/".join([sup.SPACK_PATH_PADDING_CHARS] * 200) - padding_string = "[padded-to-%d-chars]" % len(long_path) - - # test filtering when padding is enabled - with spack.config.override("config:install_tree", {"padded_length": 256}): - # tty.msg with filtering on the first argument - with sup.filter_padding(): - tty.msg("here is a long path: %s/with/a/suffix" % long_path) - out, err = capfd.readouterr() - assert padding_string in out - - # tty.msg with filtering on a laterargument - with sup.filter_padding(): - tty.msg("here is a long path:", "%s/with/a/suffix" % long_path) - out, err = capfd.readouterr() - assert padding_string in out - - # tty.error with filtering on the first argument - with sup.filter_padding(): - tty.error("here is a long path: %s/with/a/suffix" % long_path) - out, err = capfd.readouterr() - assert padding_string in err - - # tty.error with filtering on a later argument - with sup.filter_padding(): - tty.error("here is a long path:", "%s/with/a/suffix" % long_path) - out, err = capfd.readouterr() - assert padding_string in err - - # test no filtering - tty.msg("here is a long path: %s/with/a/suffix" % long_path) + +@pytest.mark.not_on_windows("Padding functionality unsupported on Windows") +def test_output_filtering(capfd, install_mockery, mutable_config): + """Test filtering padding out of tty messages.""" + long_path = "/" + "/".join([sup.SPACK_PATH_PADDING_CHARS] * 200) + padding_string = "[padded-to-%d-chars]" % len(long_path) + + # test filtering when padding is enabled + with spack.config.override("config:install_tree", {"padded_length": 256}): + # tty.msg with filtering on the first argument + with sup.filter_padding(): + tty.msg("here is a long path: %s/with/a/suffix" % long_path) + out, err = capfd.readouterr() + assert padding_string in out + + # tty.msg with filtering on a laterargument + with sup.filter_padding(): + tty.msg("here is a long path:", "%s/with/a/suffix" % long_path) + out, err = capfd.readouterr() + assert padding_string in out + + # tty.error with filtering on the first argument + with sup.filter_padding(): + tty.error("here is a long path: %s/with/a/suffix" % long_path) + out, err = capfd.readouterr() + assert padding_string in err + + # tty.error with filtering on a later argument + with sup.filter_padding(): + tty.error("here is a long path:", "%s/with/a/suffix" % long_path) out, err = capfd.readouterr() - assert padding_string not in out - - def test_pad_on_path_sep_boundary(self): - """Ensure that padded paths do not end with path separator.""" - pad_length = len(sup.SPACK_PATH_PADDING_CHARS) - padded_length = 128 - remainder = padded_length % (pad_length + 1) - path = "a" * (remainder - 1) - result = sup.add_padding(path, padded_length) - assert 128 == len(result) and not result.endswith(os.path.sep) + assert padding_string in err + + # test no filtering + tty.msg("here is a long path: %s/with/a/suffix" % long_path) + out, err = capfd.readouterr() + assert padding_string not in out + + +@pytest.mark.not_on_windows("Padding functionality unsupported on Windows") +def test_pad_on_path_sep_boundary(): + """Ensure that padded paths do not end with path separator.""" + pad_length = len(sup.SPACK_PATH_PADDING_CHARS) + padded_length = 128 + remainder = padded_length % (pad_length + 1) + path = "a" * (remainder - 1) + result = sup.add_padding(path, padded_length) + assert 128 == len(result) and not result.endswith(os.path.sep) @pytest.mark.parametrize("debug", [1, 2]) diff --git a/lib/spack/spack/test/util/remote_file_cache.py b/lib/spack/spack/test/util/remote_file_cache.py index 4c5e81332acae9..b7f8188f06d9aa 100644 --- a/lib/spack/spack/test/util/remote_file_cache.py +++ b/lib/spack/spack/test/util/remote_file_cache.py @@ -94,16 +94,15 @@ def _has_content(filename): tty.debug(f"Expected {element} in '{filename}'") return False - def _dest_dir(): - return join_path(str(tmp_path), "cache") + dest_dir = join_path(str(tmp_path), "cache") if err is not None: with spack.config.override("config:url_fetch_method", "curl"): with pytest.raises(err, match=msg): - rfc_util.local_path(url, sha256, _dest_dir) + rfc_util.local_path(url, sha256, dest_dir) else: with spack.config.override("config:url_fetch_method", "curl"): - path = rfc_util.local_path(url, sha256, _dest_dir) + path = rfc_util.local_path(url, sha256, dest_dir) assert os.path.exists(path) # Ensure correct file is "fetched" assert os.path.basename(path) == os.path.basename(url) diff --git a/lib/spack/spack/test/util/spack_lock_wrapper.py b/lib/spack/spack/test/util/spack_lock_wrapper.py index dc252103355738..793b61849c3db6 100644 --- a/lib/spack/spack/test/util/spack_lock_wrapper.py +++ b/lib/spack/spack/test/util/spack_lock_wrapper.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for Spack's wrapper module around spack.llnl.util.lock.""" + import os import pathlib diff --git a/lib/spack/spack/test/util/spack_yaml.py b/lib/spack/spack/test/util/spack_yaml.py index 52809e67fb9a75..cef0dfaacf8a99 100644 --- a/lib/spack/spack/test/util/spack_yaml.py +++ b/lib/spack/spack/test/util/spack_yaml.py @@ -21,7 +21,7 @@ def check_blame(element, file_name, line=None): """Check that `config blame config` gets right file/line for an element. This runs `spack config blame config` and scrapes the output for a - particular YAML key. It thne checks that the requested file/line info + particular YAML key. It then checks that the requested file/line info is also on that line. Line is optional; if it is ``None`` we just check for the diff --git a/lib/spack/spack/test/util/timer.py b/lib/spack/spack/test/util/timer.py index 1523dfe308a1fb..ea5a096402ab74 100644 --- a/lib/spack/spack/test/util/timer.py +++ b/lib/spack/spack/test/util/timer.py @@ -10,7 +10,7 @@ class Tick: """Timer that increments the seconds passed by 1 - everytime tick is called.""" + every time tick is called.""" def __init__(self): self.time = 0.0 diff --git a/lib/spack/spack/test/util/util_url.py b/lib/spack/spack/test/util/util_url.py index af44ec8b28dbe9..dd1975677c9d8e 100644 --- a/lib/spack/spack/test/util/util_url.py +++ b/lib/spack/spack/test/util/util_url.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Test Spack's URL handling utility functions.""" + import os import pathlib import urllib.parse diff --git a/lib/spack/spack/test/utilities.py b/lib/spack/spack/test/utilities.py index c5ddae6f4a827b..3df7bf2f9efb86 100644 --- a/lib/spack/spack/test/utilities.py +++ b/lib/spack/spack/test/utilities.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Non-fixture utilities for test code. Must be imported.""" + from spack.main import make_argument_parser diff --git a/lib/spack/spack/test/variant.py b/lib/spack/spack/test/variant.py index a6de9fc91eb9aa..1f6d20f338b4e4 100644 --- a/lib/spack/spack/test/variant.py +++ b/lib/spack/spack/test/variant.py @@ -66,7 +66,7 @@ def test_satisfies(self): assert not a.satisfies(c) and not c.satisfies(a) # SingleValuedVariant and MultiValuedVariant with the same single concrete value do satisfy - # eachother + # each other b_sv = SingleValuedVariant("foo", "bar") assert b.satisfies(b_sv) and b_sv.satisfies(b) d_sv = SingleValuedVariant("foo", True) @@ -416,7 +416,7 @@ def test_str(self): class TestVariantMapTest: def test_invalid_values(self) -> None: # Value with invalid type - a = VariantMap(Spec()) + a = VariantMap() with pytest.raises(TypeError): a["foo"] = 2 @@ -437,7 +437,7 @@ def test_invalid_values(self) -> None: def test_set_item(self) -> None: # Check that all the three types of variants are accepted - a = VariantMap(Spec()) + a = VariantMap() a["foo"] = BoolValuedVariant("foo", True) a["bar"] = SingleValuedVariant("bar", "baz") @@ -445,7 +445,7 @@ def test_set_item(self) -> None: def test_substitute(self) -> None: # Check substitution of a key that exists - a = VariantMap(Spec()) + a = VariantMap() a["foo"] = BoolValuedVariant("foo", True) a.substitute(SingleValuedVariant("foo", "bar")) @@ -456,34 +456,34 @@ def test_substitute(self) -> None: def test_satisfies_and_constrain(self) -> None: # foo=bar foobar=fee feebar=foo - a = VariantMap(Spec()) - a["foo"] = MultiValuedVariant("foo", ("bar",)) - a["foobar"] = SingleValuedVariant("foobar", "fee") - a["feebar"] = SingleValuedVariant("feebar", "foo") + a = Spec() + a.variants["foo"] = MultiValuedVariant("foo", ("bar",)) + a.variants["foobar"] = SingleValuedVariant("foobar", "fee") + a.variants["feebar"] = SingleValuedVariant("feebar", "foo") # foo=bar,baz foobar=fee shared=True - b = VariantMap(Spec()) - b["foo"] = MultiValuedVariant("foo", ("bar", "baz")) - b["foobar"] = SingleValuedVariant("foobar", "fee") - b["shared"] = BoolValuedVariant("shared", True) + b = Spec() + b.variants["foo"] = MultiValuedVariant("foo", ("bar", "baz")) + b.variants["foobar"] = SingleValuedVariant("foobar", "fee") + b.variants["shared"] = BoolValuedVariant("shared", True) # concrete, different values do not intersect / satisfy each other assert not a.intersects(b) and not b.intersects(a) assert not a.satisfies(b) and not b.satisfies(a) # foo=bar,baz foobar=fee feebar=foo shared=True - c = VariantMap(Spec()) - c["foo"] = MultiValuedVariant("foo", ("bar", "baz")) - c["foobar"] = SingleValuedVariant("foobar", "fee") - c["feebar"] = SingleValuedVariant("feebar", "foo") - c["shared"] = BoolValuedVariant("shared", True) + c = Spec() + c.variants["foo"] = MultiValuedVariant("foo", ("bar", "baz")) + c.variants["foobar"] = SingleValuedVariant("foobar", "fee") + c.variants["feebar"] = SingleValuedVariant("feebar", "foo") + c.variants["shared"] = BoolValuedVariant("shared", True) # concrete values cannot be constrained with pytest.raises(spack.variant.UnsatisfiableVariantSpecError): - a.constrain(b) + a._constrain_variants(b) def test_copy(self) -> None: - a = VariantMap(Spec()) + a = VariantMap() a["foo"] = BoolValuedVariant("foo", True) a["bar"] = SingleValuedVariant("bar", "baz") a["foobar"] = MultiValuedVariant("foobar", ("a", "b", "c", "d", "e")) @@ -492,7 +492,7 @@ def test_copy(self) -> None: assert a == c def test_str(self) -> None: - c = VariantMap(Spec()) + c = VariantMap() c["foo"] = MultiValuedVariant("foo", ("bar", "baz")) c["foobar"] = SingleValuedVariant("foobar", "fee") c["feebar"] = SingleValuedVariant("feebar", "foo") diff --git a/lib/spack/spack/test/verification.py b/lib/spack/spack/test/verification.py index bdbcedff1b982a..c5a60a7e2f0b3c 100644 --- a/lib/spack/spack/test/verification.py +++ b/lib/spack/spack/test/verification.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for the `spack.verify` module""" + import os import pathlib import shutil diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index af103aac7d4ef8..dce64f0b9a9c9a 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -6,6 +6,7 @@ We try to maintain compatibility with RPM's version semantics where it makes sense. """ + import os import pathlib @@ -17,6 +18,7 @@ import spack.version from spack.llnl.util.filesystem import working_dir from spack.version import ( + ClosedOpenRange, EmptyRangeError, GitVersion, StandardVersion, @@ -598,6 +600,22 @@ def check_repr_and_str(vrs): check_repr_and_str("R2016a.2-3_4") +def test_str_and_hash_version_range(): + """Test that precomputed string and hash values are consistent with computed ones.""" + x = ver("1.2:3.4") + assert isinstance(x, ClosedOpenRange) + # Test that precomputed str() and hash() are assigned + assert x._string is not None and x._hash is not None + _str = str(x) + _hash = hash(x) + assert x._string == _str and x._hash == _hash + # Ensure computed values match precomputed ones + x._string = None + x._hash = None + assert _str == str(x) + assert _hash == hash(x) + + @pytest.mark.parametrize( "version_str", ["1.2string3", "1.2-3xyz_4-alpha.5", "1.2beta", "1_x_rc-4"] ) @@ -917,7 +935,7 @@ def test_git_versions_without_explicit_reference( def test_total_order_versions_and_ranges(): # The set of version ranges and individual versions are comparable, which is used in - # VersionList. The comparsion across types is based on default version comparsion + # VersionList. The comparison across types is based on default version comparison # of StandardVersion, GitVersion.ref_version, and ClosedOpenRange.lo. # StandardVersion / GitVersion (at equal ref version) diff --git a/lib/spack/spack/test/views.py b/lib/spack/spack/test/views.py index 0728e04b2dd792..d6ffdd289a108f 100644 --- a/lib/spack/spack/test/views.py +++ b/lib/spack/spack/test/views.py @@ -12,6 +12,7 @@ from spack.filesystem_view import SimpleFilesystemView, YamlFilesystemView from spack.installer import PackageInstaller from spack.spec import Spec +from spack.test.conftest import FsTree def test_remove_extensions_ordered(install_mockery, mock_fetch, tmp_path: pathlib.Path): @@ -66,3 +67,86 @@ def pkg_a_add_files_to_view(view, merge_map, skip_if_exists=True): view.add_specs(a, b) assert os.path.lexists(os.path.join(view_dir, "file")) assert os.path.lexists(os.path.join(view_dir, "subdir", "file")) + + +def test_view_unique_subdir_becomes_dir_symlink(mock_packages, tmp_path: pathlib.Path): + """With link_dirs=True, if a directory is only contributed to by a single spec, the view + should create a symlink to that directory instead of linking individual files.""" + view_dir = str(tmp_path / "view") + os.mkdir(view_dir) + + layout = DirectoryLayout(view_dir) + view = SimpleFilesystemView(view_dir, layout, link_type="symlink", link_dirs=True) + + a = Spec("pkg-a") + b = Spec("pkg-b") + a.set_prefix(str(tmp_path / "a")) + b.set_prefix(str(tmp_path / "b")) + a._mark_concrete() + b._mark_concrete() + + FsTree( + tmp_path, + { + # metadata dirs for both + "a/.spack": FsTree.dir(), + "b/.spack": FsTree.dir(), + # shared dir "lib" with different files in each + "a/lib/liba.so": FsTree.file(), + "b/lib/libb.so": FsTree.file(), + # unique dir "include/a" and "include/b" with nested content + "a/include/a/a.h": FsTree.file(), + "b/include/b/b.h": FsTree.file(), + # unique dir "bin" but at depth 0, so not deep enough to be symlinked + "a/bin/a": FsTree.file(), + }, + ) + + view.add_specs(a, b) + + # Shared dir "lib" should be a real directory with individual file symlinks + lib_dir = os.path.join(view_dir, "lib") + assert os.path.isdir(lib_dir) and not os.path.islink(lib_dir) + assert os.path.islink(os.path.join(lib_dir, "liba.so")) + assert os.path.islink(os.path.join(lib_dir, "libb.so")) + + # Unique dir "include/a" should be a directory symlink + include_link = os.path.join(view_dir, "include", "a") + assert os.path.islink(include_link) + assert os.path.isdir(include_link) + assert os.path.isfile(os.path.join(include_link, "a.h")) + + # Unique dir "include/b" should be a directory symlink pointing to b's include + include_b_link = os.path.join(view_dir, "include", "b") + assert os.path.islink(include_b_link) + assert os.path.isdir(include_b_link) + assert os.path.isfile(os.path.join(include_b_link, "b.h")) + + # Unique dir "bin/" is too shallow to be symlinked, so should be an actual dir. + assert os.path.islink(os.path.join(view_dir, "bin")) is False + assert os.path.isdir(os.path.join(view_dir, "bin")) + assert os.path.islink(os.path.join(view_dir, "bin", "a")) + + +def test_view_no_dir_symlinks(mock_packages, tmp_path: pathlib.Path): + """With link_dirs=False, no directies are symlinked.""" + view_dir = str(tmp_path / "view") + os.mkdir(view_dir) + + layout = DirectoryLayout(view_dir) + view = SimpleFilesystemView(view_dir, layout, link_type="symlink", link_dirs=False) + + a = Spec("pkg-a") + a.set_prefix(str(tmp_path / "a")) + a._mark_concrete() + + FsTree(tmp_path, {"a/.spack": FsTree.dir(), "a/include/a/a.h": FsTree.file("header")}) + + view.add_specs(a) + + # "include/a" should be a real directory, not a symlink + include_dir = os.path.join(view_dir, "include", "a") + assert os.path.isdir(include_dir) and not os.path.islink(include_dir) + # File should be a symlink. + ah_path = os.path.join(include_dir, "a.h") + assert os.path.islink(ah_path) diff --git a/lib/spack/spack/test/web.py b/lib/spack/spack/test/web.py index 3a5d91b6f9a2ea..5f6c6d6345e201 100644 --- a/lib/spack/spack/test/web.py +++ b/lib/spack/spack/test/web.py @@ -450,3 +450,64 @@ def test_ssl_curl_cert_file( assert dump_env["CURL_CA_BUNDLE"] == mock_cert else: assert "CURL_CA_BUNDLE" not in dump_env + + +@pytest.mark.parametrize( + "error_code,num_errors,max_retries,expect_failure", + [ + (500, 2, 5, False), # transient, enough retries + (500, 2, 2, True), # transient, not enough retries + (429, 2, 5, False), # rate limit, enough retries + (404, 1, 5, True), # not transient, never retried + ], +) +def test_retry_on_transient_error(error_code, num_errors, max_retries, expect_failure): + import urllib.error + + call_count = 0 + sleep_times = [] + + def flaky_func(): + nonlocal call_count + call_count += 1 + if call_count <= num_errors: + raise urllib.error.HTTPError( + url="https://example.com", code=error_code, msg="err", hdrs={}, fp=None + ) + return "ok" + + retrying = spack.util.web.retry_on_transient_error( + flaky_func, retries=max_retries, sleep=sleep_times.append + ) + + if expect_failure: + with pytest.raises(urllib.error.HTTPError): + retrying() + else: + assert retrying() == "ok" + assert sleep_times == [2**i for i in range(num_errors)] + + +def test_retry_on_transient_error_non_oserror(): + """Non-OSError exceptions with transient names (e.g. botocore) should be retried.""" + + class ResponseStreamingError(Exception): + pass + + call_count = 0 + sleep_times = [] + + def flaky_func(): + nonlocal call_count + call_count += 1 + if call_count <= 2: + raise ResponseStreamingError("IncompleteRead") + return "ok" + + retrying = spack.util.web.retry_on_transient_error( + flaky_func, retries=5, sleep=sleep_times.append + ) + + assert retrying() == "ok" + assert call_count == 3 + assert sleep_times == [1, 2] diff --git a/lib/spack/spack/tokenize.py b/lib/spack/spack/tokenize.py index 4a0a617a92d732..741b1700c0b155 100644 --- a/lib/spack/spack/tokenize.py +++ b/lib/spack/spack/tokenize.py @@ -4,6 +4,7 @@ """This module provides building blocks for tokenizing strings. Users can define tokens by inheriting from TokenBase and defining tokens as ordered enum members. The Tokenizer class can then be used to iterate over tokens in a string.""" + import enum import re from typing import Generator, Match, Optional, Type diff --git a/lib/spack/spack/traverse.py b/lib/spack/spack/traverse.py index f0f960d408a552..2975fdc71b63f4 100644 --- a/lib/spack/spack/traverse.py +++ b/lib/spack/spack/traverse.py @@ -183,7 +183,7 @@ def neighbors(self, item: EdgeAndDepth) -> List[EdgeAndDepth]: edges = item.edge.spec.edges_to_dependencies(depflag=follow) - # filter direct_type edges already followed before becuase they were also transitive_type. + # filter direct_type edges already followed before because they were also transitive_type. if seen: edges = [edge for edge in edges if not edge.depflag & self.transitive_type] @@ -200,7 +200,7 @@ def get_visitor_from_args( cover (str): Determines how extensively to cover the dag. Possible values: ``nodes`` -- Visit each unique node in the dag only once. ``edges`` -- If a node has been visited once but is reached along a - new path, it's accepted, but not recurisvely followed. This traverses + new path, it's accepted, but not recursively followed. This traverses each 'edge' in the DAG once. ``paths`` -- Explore every unique path reachable from the root. This descends into visited subtrees and will accept nodes multiple @@ -246,7 +246,7 @@ def traverse_depth_first_edges_generator(edges, visitor, post_order=False, root= Arguments: edges (list): List of EdgeAndDepth instances - visitor: class instance implementing accept() and neigbors() + visitor: class instance implementing accept() and neighbors() post_order (bool): Whether to yield nodes when backtracking root (bool): whether to yield at depth 0 depth (bool): when ``True`` yield a tuple of depth and edge, otherwise only the @@ -514,7 +514,7 @@ def traverse_edges( cover: Determines how extensively to cover the dag. Possible values: ``nodes`` -- Visit each unique node in the dag only once. ``edges`` -- If a node has been visited once but is reached along a new path, it's - accepted, but not recurisvely followed. This traverses each 'edge' in the DAG once. + accepted, but not recursively followed. This traverses each 'edge' in the DAG once. ``paths`` -- Explore every unique path reachable from the root. This descends into visited subtrees and will accept nodes multiple times if they're reachable by multiple paths. @@ -626,7 +626,7 @@ def traverse_nodes( cover: Determines how extensively to cover the dag. Possible values: ``nodes`` -- Visit each unique node in the dag only once. ``edges`` -- If a node has been visited once but is reached along a new path, it's - accepted, but not recurisvely followed. This traverses each 'edge' in the DAG once. + accepted, but not recursively followed. This traverses each 'edge' in the DAG once. ``paths`` -- Explore every unique path reachable from the root. This descends into visited subtrees and will accept nodes multiple times if they're reachable by multiple paths. @@ -673,7 +673,7 @@ def traverse_tree( cover: Determines how extensively to cover the dag. Possible values: ``nodes`` -- Visit each unique node in the dag only once. ``edges`` -- If a node has been visited once but is reached along a - new path, it's accepted, but not recurisvely followed. This traverses each 'edge' in + new path, it's accepted, but not recursively followed. This traverses each 'edge' in the DAG once. ``paths`` -- Explore every unique path reachable from the root. This descends into visited subtrees and will accept nodes multiple times if they're reachable by multiple diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index 0aabc7e40d5525..711d49d05417c9 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -28,6 +28,7 @@ spack doesn't need anyone to tell it where to get the tarball even though it's never been told about that version before. """ + import io import os import pathlib @@ -169,7 +170,7 @@ def parse_version_offset(path: str) -> Tuple[str, int, int, int, str]: # ] # # The first regex that matches string will be used to determine - # the version of the package. Thefore, hyperspecific regexes should + # the version of the package. Therefore, hyperspecific regexes should # come first while generic, catch-all regexes should come last. # With that said, regular expressions are slow, so if possible, put # ones that only catch one or two URLs at the bottom. @@ -357,7 +358,7 @@ def parse_name_offset( # ] # # The first regex that matches string will be used to determine - # the name of the package. Thefore, hyperspecific regexes should + # the name of the package. Therefore, hyperspecific regexes should # come first while generic, catch-all regexes should come last. # With that said, regular expressions are slow, so if possible, put # ones that only catch one or two URLs at the bottom. @@ -531,7 +532,7 @@ def substitute_version(path: str, new_version) -> str: >>> substitute_version("https://www.hdfgroup.org/ftp/HDF/releases/HDF4.2.12/src/hdf-4.2.12.tar.gz", "2.3") "https://www.hdfgroup.org/ftp/HDF/releases/HDF2.3/src/hdf-2.3.tar.gz" - """ + """ # noqa: E501 (name, ns, nl, noffs, ver, vs, vl, voffs) = substitution_offsets(path) new_path = "" diff --git a/lib/spack/spack/url_buildcache.py b/lib/spack/spack/url_buildcache.py index 24a3c0cca61822..561733e15f012f 100644 --- a/lib/spack/spack/url_buildcache.py +++ b/lib/spack/spack/url_buildcache.py @@ -159,7 +159,7 @@ class URLBuildcacheEntry: This class manages access to a versioned buildcache entry by providing a means to download both the metadata (spec file) and compressed archive. - It also provides methods for accessing the paths/urls associcated with + It also provides methods for accessing the paths/urls associated with buildcache entries. Starting with buildcache layout version 3, it is not possible to know @@ -448,7 +448,7 @@ def read_manifest(self, manifest_url: Optional[str] = None) -> BuildcacheManifes if self.manifest: if not manifest_url or manifest_url == self.remote_manifest_url: # We already have a manifest, so now calling this method without a specific - # manifiest url, or with the same one we have internally, then skip reading + # manifest url, or with the same one we have internally, then skip reading # again, and just return the manifest we already read. return self.manifest @@ -465,8 +465,7 @@ def read_manifest(self, manifest_url: Optional[str] = None) -> BuildcacheManifes manifest_contents = "" try: - _, _, manifest_file = web_util.read_from_url(manifest_url) - manifest_contents = io.TextIOWrapper(manifest_file, encoding="utf-8").read() + manifest_contents = web_util.read_text(manifest_url) except (web_util.SpackWebError, OSError) as e: raise BuildcacheEntryError(f"Error reading manifest at {manifest_url}") from e @@ -1110,6 +1109,8 @@ def file_read_method(manifest_path: str) -> URLBuildcacheEntry: include_pattern = cache_class.get_buildcache_component_include_pattern(component_type) component_prefix = cache_class.get_relative_path_components(component_type) + component_url = url_util.join(url, *component_prefix) + sync_command_args = [ "s3", "sync", @@ -1117,17 +1118,17 @@ def file_read_method(manifest_path: str) -> URLBuildcacheEntry: "*", "--include", include_pattern, - url_util.join(url, *component_prefix), + component_url, tmpspecsdir, ] # Use aws s3 ls to get mtimes of manifests - ls_command_args = ["s3", "ls", "--recursive", url] + ls_command_args = ["s3", "ls", "--recursive", component_url] s3_ls_regex = re.compile(r"^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\s+\d+\s+(.+)$") filename_to_mtime: Dict[str, float] = {} - tty.debug(f"Using aws s3 sync to download manifests from {url} to {tmpspecsdir}") + tty.debug(f"Using aws s3 sync to download manifests from {component_url} to {tmpspecsdir}") try: aws(*sync_command_args, output=os.devnull, error=os.devnull) @@ -1353,7 +1354,7 @@ def try_verify(specfile_path): class MirrorMetadata: """Simple class to hold a mirror url and a buildcache layout version - This class is used by BinaryCacheIndex to produce a key used to keep + This class is used by BinaryIndexCache to produce a key used to keep track of downloaded/processed buildcache index files from remote mirrors in some layout version.""" @@ -1379,7 +1380,7 @@ def __hash__(self): return hash((self.url, self.version, self.view)) @classmethod - def from_string(cls, s: str): + def from_string(cls, s: str) -> "MirrorMetadata": m = re.match(r"^(.*)__v([0-9]+)(?:__(.*))?$", s) if not m: raise MirrorMetadataError(f"Malformed string {s}") diff --git a/lib/spack/spack/user_environment.py b/lib/spack/spack/user_environment.py index dca1e5437f1011..5dfd50f311f684 100644 --- a/lib/spack/spack/user_environment.py +++ b/lib/spack/spack/user_environment.py @@ -116,21 +116,4 @@ def environment_modifications_for_specs( if view: project_env_mods(*topo_ordered, view=view, env=env) - # we don't want to set PYTHONPATH to the default search path in virtual environments - view_python_pattern = re.compile( - r"^" + re.escape(os.path.join(view.root, "lib")) + r"/python[^/]+/site-packages$" - ) - - mods = [ - mod.value - for mod in env.env_modifications - if ( - isinstance(mod, environment.PrependPath) - and mod.name == "PYTHONPATH" - and view_python_pattern.match(mod.value) - ) - ] - - for modif in mods: - env.remove_path("PYTHONPATH", modif) return env diff --git a/lib/spack/spack/util/archive.py b/lib/spack/spack/util/archive.py index 99c155f1b21112..c96ce32619e79a 100644 --- a/lib/spack/spack/util/archive.py +++ b/lib/spack/spack/util/archive.py @@ -101,7 +101,7 @@ def gzip_compressed_tarfile( path: str, ) -> Generator[Tuple[tarfile.TarFile, ChecksumWriter, ChecksumWriter], None, None]: """Create a reproducible, gzip compressed tarfile, and keep track of shasums of both the - compressed and uncompressed tarfile. Reproduciblity is achived by normalizing the gzip header + compressed and uncompressed tarfile. Reproducibility is achieved by normalizing the gzip header (no file name and zero mtime). Yields: @@ -114,7 +114,7 @@ def gzip_compressed_tarfile( # Create gzip compressed tarball of the install prefix # 1) Use explicit empty filename and mtime 0 for gzip header reproducibility. # If the filename="" is dropped, Python will use fileobj.name instead. - # This should effectively mimick `gzip --no-name`. + # This should effectively mimic `gzip --no-name`. # 2) On AMD Ryzen 3700X and an SSD disk, we have the following on compression speed: # compresslevel=6 gzip default: llvm takes 4mins, roughly 2.1GB # compresslevel=9 python default: llvm takes 12mins, roughly 2.1GB @@ -259,7 +259,7 @@ def retrieve_commit_from_archive(archive_path, ref): try: with tarfile.open(archive_path, "r") as tar: names = tar.getnames() - # since we always have a prefix and can't gaurantee the value we need this lookup. + # since we always have a prefix and can't guarantee the value we need this lookup. prefix = "" for name in names: if name.endswith(".git"): diff --git a/lib/spack/spack/util/compression.py b/lib/spack/spack/util/compression.py index dc7ba72ca95d1a..32f360fb6eb9db 100644 --- a/lib/spack/spack/util/compression.py +++ b/lib/spack/spack/util/compression.py @@ -128,7 +128,7 @@ def _gunzip(archive_file: str) -> str: def _py_gunzip(archive_file: str) -> str: - """Returns path to gunzip'd file. Decompresses `.gz` compressed archvies via python gzip + """Returns path to gunzip'd file. Decompresses `.gz` compressed archives via python gzip module""" decompressed_file = os.path.basename( spack.llnl.url.strip_compression_extension(archive_file, "gz") @@ -522,7 +522,7 @@ def extension_from_magic_numbers_by_stream( """Returns the typical extension for the opened file, without leading ``.``, based on its magic numbers. - If the stream does not represent file type recongized by Spack (see + If the stream does not represent file type recognized by Spack (see :py:data:`SUPPORTED_FILETYPES`), the method will return None Args: diff --git a/lib/spack/spack/util/cpus.py b/lib/spack/spack/util/cpus.py index feabcc10798724..9676120c8b0365 100644 --- a/lib/spack/spack/util/cpus.py +++ b/lib/spack/spack/util/cpus.py @@ -9,7 +9,7 @@ def cpus_available(): """ Returns the number of CPUs available for the current process, or the number - of phyiscal CPUs when that information cannot be retrieved. The number + of physical CPUs when that information cannot be retrieved. The number of available CPUs might differ from the number of physical CPUs when using spack through Slurm or container runtimes. """ diff --git a/lib/spack/spack/util/crypto.py b/lib/spack/spack/util/crypto.py index 9d514f6c514043..6019ac24721dd2 100644 --- a/lib/spack/spack/util/crypto.py +++ b/lib/spack/spack/util/crypto.py @@ -14,7 +14,7 @@ # Note: keys are ordered by popularity for earliest return in ``hash_key in version_dict`` checks. -#: size of hash digests in bytes, mapped to algoritm names +#: size of hash digests in bytes, mapped to algorithm names _size_to_hash = dict((v, k) for k, v in hashes.items()) diff --git a/lib/spack/spack/util/ctest_log_parser.py b/lib/spack/spack/util/ctest_log_parser.py index f8cfe16f2ae14f..421d68121aff98 100644 --- a/lib/spack/spack/util/ctest_log_parser.py +++ b/lib/spack/spack/util/ctest_log_parser.py @@ -68,15 +68,14 @@ up to date with CTest, just make sure the ``*_matches`` and ``*_exceptions`` lists are kept up to date with CTest's build handler. """ + import io -import math -import multiprocessing import re -import sys -import threading import time -from contextlib import contextmanager -from typing import List, Optional, TextIO, Tuple, Union +from collections import deque +from typing import Dict, Iterable, List, Optional, TextIO, Tuple, Union + +from spack.llnl.util.lang import PatternStr _error_matches = [ "^FAIL: ", @@ -87,21 +86,22 @@ "^[Bb]us [Ee]rror", "^[Ss]egmentation [Vv]iolation", "^[Ss]egmentation [Ff]ault", - ":.*[Pp]ermission [Dd]enied", - "[^ :]:[0-9]+: [^ \\t]", - "[^:]: error[ \\t]*[0-9]+[ \\t]*:", + "Permission [Dd]enied", + "permission [Dd]enied", + ":[0-9]+: [^ \\t]", + ": error[ \\t]*[0-9]+[ \\t]*:", "^Error ([0-9]+):", "^Fatal", "^[Ee]rror: ", "^Error ", - "[0-9] ERROR: ", + " ERROR: ", '^"[^"]+", line [0-9]+: [^Ww]', "^cc[^C]*CC: ERROR File = ([^,]+), Line = ([0-9]+)", "^ld([^:])*:([ \\t])*ERROR([^:])*:", "^ild:([ \\t])*\\(undefined symbol\\)", - "[^ :] : (error|fatal error|catastrophic error)", - "[^:]: (Error:|error|undefined reference|multiply defined)", - "[^:]\\([^\\)]+\\) ?: (error|fatal error|catastrophic error)", + ": (error|fatal error|catastrophic error)", + ": (Error:|error|undefined reference|multiply defined)", + "\\([^\\)]+\\) ?: (error|fatal error|catastrophic error)", "^fatal error C[0-9]+:", ": syntax error ", "^collect2: ld returned 1 exit status", @@ -152,28 +152,27 @@ " ok", "Note:", ":[ \\t]+Where:", - "[^ :]:[0-9]+: Warning", + ":[0-9]+: Warning", "------ Build started: .* ------", ] #: Regexes to match file/line numbers in error/warning messages _warning_matches = [ - "[^ :]:[0-9]+: warning:", - "[^ :]:[0-9]+: note:", + ":[0-9]+: warning:", + ":[0-9]+: note:", "^cc[^C]*CC: WARNING File = ([^,]+), Line = ([0-9]+)", "^ld([^:])*:([ \\t])*WARNING([^:])*:", - "[^:]: warning [0-9]+:", + ": warning [0-9]+:", '^"[^"]+", line [0-9]+: [Ww](arning|arnung)', - "[^:]: warning[ \\t]*[0-9]+[ \\t]*:", + ": warning[ \\t]*[0-9]+[ \\t]*:", "^(Warning|Warnung) ([0-9]+):", "^(Warning|Warnung)[ :]", "WARNING: ", - "[^ :] : warning", - "[^:]: warning", + ": warning", '", line [0-9]+\\.[0-9]+: [0-9]+-[0-9]+ \\([WI]\\)', "^cxx: Warning:", "file: .* has no symbols", - "[^ :]:[0-9]+: (Warning|Warnung)", + ":[0-9]+: (Warning|Warnung)", "\\([0-9]*\\): remark #[0-9]*", '".*", line [0-9]+: remark\\([0-9]*\\):', "cc-[0-9]* CC: REMARK File = .*, Line = [0-9]*", @@ -214,43 +213,46 @@ class LogEvent: """Class representing interesting events (e.g., errors) in a build log.""" + #: color name when rendering in the terminal + color = "" + def __init__( self, - text, - line_no, - source_file=None, - source_line_no=None, - pre_context=None, - post_context=None, - ): + text: str, + line_no: int, + source_file: Optional[str] = None, + source_line_no: Optional[str] = None, + pre_context: Optional[List[str]] = None, + post_context: Optional[List[str]] = None, + ) -> None: self.text = text self.line_no = line_no - self.source_file = (source_file,) - self.source_line_no = (source_line_no,) + self.source_file = source_file + self.source_line_no = source_line_no self.pre_context = pre_context if pre_context is not None else [] self.post_context = post_context if post_context is not None else [] self.repeat_count = 0 @property - def start(self): + def start(self) -> int: """First line in the log with text for the event or its context.""" return self.line_no - len(self.pre_context) @property - def end(self): + def end(self) -> int: """Last line in the log with text for event or its context.""" return self.line_no + len(self.post_context) + 1 - def __getitem__(self, line_no): + def __getitem__(self, line_no: int) -> str: """Index event text and context by actual line number in file.""" if line_no == self.line_no: return self.text elif line_no < self.line_no: return self.pre_context[line_no - self.line_no] - elif line_no > self.line_no: + else: return self.post_context[line_no - self.line_no - 1] - def __str__(self): + def __str__(self) -> str: """Returns event lines and context.""" out = io.StringIO() for i in range(self.start, self.end): @@ -264,185 +266,212 @@ def __str__(self): class BuildError(LogEvent): """LogEvent subclass for build errors.""" + color = "R" + class BuildWarning(LogEvent): """LogEvent subclass for build warnings.""" - -def chunks(xs, n): - """Divide xs into n approximately-even chunks.""" - chunksize = int(math.ceil(len(xs) / n)) - return [xs[i : i + chunksize] for i in range(0, len(xs), chunksize)] + color = "Y" + + +def _optimize_regexes(regex_strings: List[str]) -> List[str]: + """Groups regexes by their first character and combines each group into a single regex using + alternation. Python's regex compiler optimizes the combined pattern to share common prefixes + internally. The result is a shorter list of regexes that all hit a fast path in cpython's regex + engine for prefix matching.""" + groups: Dict[str, List[str]] = {} + for regex in sorted(regex_strings): + key = regex[:1] # empty or single character + if key == "\\": # include escaped character + key = regex[:2] + if key not in groups: + groups[key] = [regex] + else: + groups[key].append(regex) + return ["|".join(entries) for entries in groups.values()] -@contextmanager -def _time(times, i): - start = time.time() - yield - end = time.time() - times[i] += end - start +class _Matcher: + """Tests a log line against match/exception regex lists.""" + def __init__(self, matches: List[PatternStr], exceptions: List[PatternStr]) -> None: + self.matches = matches + self.exceptions = exceptions -def _match(matches, exceptions, line): - """True if line matches a regex in matches and none in exceptions.""" - return any(m.search(line) for m in matches) and not any(e.search(line) for e in exceptions) + def __call__(self, line: str) -> bool: + """Returns True if line matches any regex in self.matches and none in self.exceptions.""" + for match in self.matches: + if match.search(line): + break + else: + return False + for exc in self.exceptions: + if exc.search(line): + return False + return True -def _profile_match(matches, exceptions, line, match_times, exc_times): - """Profiled version of match(). +class _ProfileMatcher(_Matcher): + """Variant of _Matcher that records time spent in each regex.""" - Timing is expensive so we have two whole functions. This is much - longer because we have to break up the ``any()`` calls. + def __init__(self, matches: List[PatternStr], exceptions: List[PatternStr]) -> None: + super().__init__(matches, exceptions) + self.match_times = [0.0] * len(matches) + self.exc_times = [0.0] * len(exceptions) - """ - for i, m in enumerate(matches): - with _time(match_times, i): - if m.search(line): + def __call__(self, line: str) -> bool: + for i, m in enumerate(self.matches): + start = time.perf_counter() + found = m.search(line) + self.match_times[i] += time.perf_counter() - start + if found: break - else: - return False + else: + return False - for i, m in enumerate(exceptions): - with _time(exc_times, i): - if m.search(line): + for i, m in enumerate(self.exceptions): + start = time.perf_counter() + found = m.search(line) + self.exc_times[i] += time.perf_counter() - start + if found: return False - else: return True + def print_timings(self, kind: str) -> None: + print() + print(f"{kind}_matches") + for pattern, t in zip(self.matches, self.match_times): + print("%16.2f %s" % (t * 1e6, pattern.pattern)) + print() + print(f"{kind}_exceptions") + for pattern, t in zip(self.exceptions, self.exc_times): + print("%16.2f %s" % (t * 1e6, pattern.pattern)) + + +def _parse( + stream: Iterable[str], + error_matcher: _Matcher, + warning_matcher: _Matcher, + file_line_matches: List[PatternStr], + context: int, + tail: int = 0, +) -> Tuple[List[BuildError], List[BuildWarning], Optional[LogEvent]]: + + errors: List[BuildError] = [] + warnings: List[BuildWarning] = [] + # rolling window of recent lines + pre_context: deque[str] = deque(maxlen=max(context, tail)) + # list of (event, remaining_post_context_lines) + pending_events: List[Tuple[Union[BuildError, BuildWarning], int]] = [] + + last_line_no = 0 + for i, line in enumerate(stream): + rstripped_line = line.rstrip() + last_line_no = i + 1 + + # feed this line into every event still collecting post_context + if pending_events: + active_events = [] + for event, remaining in pending_events: + event.post_context.append(rstripped_line) + if remaining > 1: + active_events.append((event, remaining - 1)) + elif isinstance(event, BuildError): + errors.append(event) + else: + warnings.append(event) + pending_events = active_events -def _parse(lines, offset, profile): - def compile(regex_array): - return [re.compile(regex) for regex in regex_array] - - error_matches = compile(_error_matches) - error_exceptions = compile(_error_exceptions) - warning_matches = compile(_warning_matches) - warning_exceptions = compile(_warning_exceptions) - file_line_matches = compile(_file_line_matches) - - matcher, _ = _match, [] - timings = [] - if profile: - matcher = _profile_match - timings = [ - [0.0] * len(error_matches), - [0.0] * len(error_exceptions), - [0.0] * len(warning_matches), - [0.0] * len(warning_exceptions), - ] - - errors = [] - warnings = [] - for i, line in enumerate(lines): # use CTest's regular expressions to scrape the log for events - if matcher(error_matches, error_exceptions, line, *timings[:2]): - event = BuildError(line.strip(), offset + i + 1) - errors.append(event) - elif matcher(warning_matches, warning_exceptions, line, *timings[2:]): - event = BuildWarning(line.strip(), offset + i + 1) - warnings.append(event) + if error_matcher(line): + event = BuildError(rstripped_line, i + 1) + elif warning_matcher(line): + event = BuildWarning(rstripped_line, i + 1) else: + pre_context.append(rstripped_line) continue - # get file/line number for each event, if possible + event.pre_context = list(pre_context)[-context:] if context else [] + event.post_context = [] + + # get file/line number for the event, if possible for flm in file_line_matches: match = flm.search(line) if match: event.source_file, event.source_line_no = match.groups() + break + + if context > 0: + pending_events.append((event, context)) + elif isinstance(event, BuildError): + errors.append(event) + else: + warnings.append(event) - return errors, warnings, timings + pre_context.append(rstripped_line) + # flush events whose post_context window extends past EOF + for event, _ in pending_events: + if isinstance(event, BuildError): + errors.append(event) + else: + warnings.append(event) -def _parse_unpack(args): - return _parse(*args) + # build tail section from the last N lines of the log, if requested + if tail > 0 and last_line_no > 0: + lines = list(pre_context)[-tail:] + tail_event = LogEvent(text=lines[-1], line_no=last_line_no, pre_context=lines[:-1]) + else: + tail_event = None + + return errors, warnings, tail_event class CTestLogParser: """Log file parser that extracts errors and warnings.""" - def __init__(self, profile=False): - # whether to record timing information - self.timings = [] - self.profile = profile - - def print_timings(self): - """Print out profile of time spent in different regular expressions.""" + def __init__(self, profile: bool = False) -> None: + error_matches = [re.compile(r) for r in _optimize_regexes(_error_matches)] + error_exceptions = [re.compile(r) for r in _optimize_regexes(_error_exceptions)] + warning_matches = [re.compile(r) for r in _optimize_regexes(_warning_matches)] + warning_exceptions = [re.compile(r) for r in _optimize_regexes(_warning_exceptions)] - def stringify(elt): - return elt if isinstance(elt, str) else elt.pattern + cls = _ProfileMatcher if profile else _Matcher + self._error_matcher = cls(error_matches, error_exceptions) + self._warning_matcher = cls(warning_matches, warning_exceptions) + self._file_line_matches = [re.compile(r) for r in _file_line_matches] - index = 0 - for name, arr in [ - ("error_matches", _error_matches), - ("error_exceptions", _error_exceptions), - ("warning_matches", _warning_matches), - ("warning_exceptions", _warning_exceptions), - ]: - - print() - print(name) - for i, elt in enumerate(arr): - print("%16.2f %s" % (self.timings[index][i] * 1e6, stringify(elt))) - index += 1 + def print_timings(self) -> None: + """Print out profile of time spent in different regular expressions.""" + assert isinstance(self._error_matcher, _ProfileMatcher) + assert isinstance(self._warning_matcher, _ProfileMatcher) + self._error_matcher.print_timings("error") + self._warning_matcher.print_timings("warning") def parse( - self, stream: Union[str, TextIO], context: int = 6, jobs: Optional[int] = None - ) -> Tuple[List[BuildError], List[BuildWarning]]: + self, stream: Union[str, TextIO, List[str]], context: int = 6, tail: int = 0 + ) -> Tuple[List[BuildError], List[BuildWarning], Optional[LogEvent]]: """Parse a log file by searching each line for errors and warnings. Args: stream: filename or stream to read from context: lines of context to extract around each log event + tail: if > 0, also return a :class:`LogEvent` with the last ``tail`` lines Returns: - two lists containing :class:`BuildError` and :class:`BuildWarning` objects. + two lists containing :class:`BuildError` and :class:`BuildWarning` objects, + plus an optional :class:`LogEvent` for the tail (None when ``tail=0``). """ if isinstance(stream, str): - with open(stream) as f: - return self.parse(f, context, jobs) - - lines = [line for line in stream] - - if jobs is None: - jobs = multiprocessing.cpu_count() - - # single-thread small logs - if len(lines) < 10 * jobs: - errors, warnings, self.timings = _parse(lines, 0, self.profile) - - else: - # Build arguments for parallel jobs - args = [] - offset = 0 - for chunk in chunks(lines, jobs): - args.append((chunk, offset, self.profile)) - offset += len(chunk) - - # create a pool and farm out the matching job - pool = multiprocessing.Pool(jobs) - try: - # this is a workaround for a Python bug in Pool with ctrl-C - if sys.version_info >= (3, 2): - max_timeout = threading.TIMEOUT_MAX - else: - max_timeout = 9999999 - results = pool.map_async(_parse_unpack, args, 1).get(max_timeout) - - errors, warnings, timings = zip(*results) - finally: - pool.terminate() - - # merge results - errors = sum(errors, []) - warnings = sum(warnings, []) - - if self.profile: - self.timings = [[sum(i) for i in zip(*t)] for t in zip(*timings)] - - # add log context to all events - for event in errors + warnings: - i = event.line_no - 1 - event.pre_context = [x.rstrip() for x in lines[i - context : i]] - event.post_context = [x.rstrip() for x in lines[i + 1 : i + context + 1]] - - return errors, warnings + with open(stream, encoding="utf-8", errors="replace") as f: + return self.parse(f, context, tail) + + return _parse( + stream, + self._error_matcher, + self._warning_matcher, + self._file_line_matches, + context, + tail, + ) diff --git a/lib/spack/spack/util/debug.py b/lib/spack/spack/util/debug.py deleted file mode 100644 index 814779070caedd..00000000000000 --- a/lib/spack/spack/util/debug.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright Spack Project Developers. See COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -"""Debug signal handler: prints a stack trace and enters interpreter. - -``register_interrupt_handler()`` enables a ctrl-C handler that prints -a stack trace and drops the user into an interpreter. - -""" -import code -import io -import os -import pdb -import signal -import sys -import traceback - - -def debug_handler(sig, frame): - """Interrupt running process, and provide a python prompt for - interactive debugging.""" - d = {"_frame": frame} # Allow access to frame object. - d.update(frame.f_globals) # Unless shadowed by global - d.update(frame.f_locals) - - i = code.InteractiveConsole(d) - message = "Signal received : entering python shell.\nTraceback:\n" - message += "".join(traceback.format_stack(frame)) - i.interact(message) - os._exit(1) # Use os._exit to avoid test harness. - - -def register_interrupt_handler(): - """Print traceback and enter an interpreter on Ctrl-C""" - signal.signal(signal.SIGINT, debug_handler) - - -# Subclass of the debugger to keep readline working. See -# https://stackoverflow.com/questions/4716533/how-to-attach-debugger-to-a-python-subproccess/23654936 -class ForkablePdb(pdb.Pdb): - """ - This class allows the python debugger to follow forked processes - and can set tracepoints allowing the Python Debugger Pdb to be used - from a python multiprocessing child process. - - This is used the same way one would normally use Pdb, simply import this - class and use as a drop in for Pdb, although the syntax here is slightly different, - requiring the instantiton of this class, i.e. ForkablePdb().set_trace(). - - This should be used when attempting to call a debugger from a - child process spawned by the python multiprocessing such as during - the run of Spack.install, or any where else Spack spawns a child process. - """ - - try: - _original_stdin_fd = sys.stdin.fileno() - except io.UnsupportedOperation: - _original_stdin_fd = None - _original_stdin = None - - def __init__(self, stdout_fd=None, stderr_fd=None): - pdb.Pdb.__init__(self, nosigint=True) - self._stdout_fd = stdout_fd - self._stderr_fd = stderr_fd - - def _cmdloop(self): - current_stdin = sys.stdin - try: - if not self._original_stdin: - self._original_stdin = os.fdopen(self._original_stdin_fd) - sys.stdin = self._original_stdin - if self._stdout_fd is not None: - os.dup2(self._stdout_fd, sys.stdout.fileno()) - os.dup2(self._stdout_fd, self.stdout.fileno()) - if self._stderr_fd is not None: - os.dup2(self._stderr_fd, sys.stderr.fileno()) - self.cmdloop() - finally: - sys.stdin = current_stdin diff --git a/lib/spack/spack/util/editor.py b/lib/spack/spack/util/editor.py index de65088b74bd7f..9c6ce82a594885 100644 --- a/lib/spack/spack/util/editor.py +++ b/lib/spack/spack/util/editor.py @@ -11,6 +11,7 @@ neither variable is set, we fall back to one of several common editors, raising an OSError if we are unable to find one. """ + import os import shlex from typing import Callable, List diff --git a/lib/spack/spack/util/elf.py b/lib/spack/spack/util/elf.py index d46b50e47235c6..d4d5e68d548ed1 100644 --- a/lib/spack/spack/util/elf.py +++ b/lib/spack/spack/util/elf.py @@ -336,7 +336,7 @@ def parse_pt_dynamic(f: BinaryIO, elf: ElfFile) -> None: except OSError: raise ElfParsingError("Could not seek to PT_DYNAMIC entry") - # In case of broken ELF files, don't read beyond the advertized size. + # In case of broken ELF files, don't read beyond the advertised size. for _ in range(elf.pt_dynamic_p_filesz // dynamic_array_size): data = read_exactly(f, dynamic_array_size, "Malformed dynamic array entry") tag, val = unpack(dynamic_array_fmt, data) diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index 82140855cd360e..598e2986341513 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Set, unset or modify environment variables.""" + import collections import contextlib import inspect @@ -22,7 +23,7 @@ from spack.llnl.util.lang import dedupe # List is invariant, so List[str] is not a subtype of List[Union[str, pathlib.PurePath]]. -# Sequence is covariant, but because str itself is a subtype of Sequence[str], we cannot exlude it +# Sequence is covariant, but because str itself is a subtype of Sequence[str], we cannot exclude it # in the type hint. So, use an awkward union type to allow (mixed) str and PurePath items. ListOfPaths = Union[List[str], List[pathlib.PurePath], List[Union[str, pathlib.PurePath]]] @@ -165,7 +166,7 @@ def dump_environment(path: Path, environment: Optional[MutableMapping[str, str]] Args: path: path of the file to write - environment: environment to be writte. If None os.environ is used. + environment: environment to be written. If None os.environ is used. """ use_env = environment or os.environ hidden_vars = {"PS1", "PWD", "OLDPWD", "TERM_SESSION_ID"} diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index f42417275647dc..817bb0563cb471 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -2,12 +2,14 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import io import os import re +import shlex import subprocess import sys from pathlib import Path, PurePath -from typing import Callable, Dict, List, Optional, Sequence, TextIO, Type, Union, overload +from typing import BinaryIO, Callable, Dict, List, Optional, Sequence, Tuple, Type, Union, overload from spack.vendor.typing_extensions import Literal @@ -17,6 +19,43 @@ __all__ = ["Executable", "which", "which_string", "ProcessError"] +OutType = Union[Optional[BinaryIO], str, Type[str], Callable] + + +def _process_cmd_output( + out: bytes, + err: bytes, + output: OutType, + error: OutType, + encoding: str = "ISO-8859-1" if sys.platform == "win32" else "utf-8", +) -> Optional[str]: + if output is str or output is str.split or error is str or error is str.split: + result = "" + if output is str or output is str.split: + outstr = out.decode(encoding) + result += outstr + if output is str.split: + sys.stdout.write(outstr) + if error is str or error is str.split: + errstr = err.decode(encoding) + result += errstr + if error is str.split: + sys.stderr.write(errstr) + return result + else: + return None + + +def _streamify_output(arg: OutType, name: str) -> Tuple[Union[int, BinaryIO, None], bool]: + if isinstance(arg, str): + return open(arg, "wb"), True + elif arg is str or arg is str.split: + return subprocess.PIPE, False + elif callable(arg): + raise ValueError(f"`{name}` must be a stream, a filename, or `str`/`str.split`") + else: + return arg, False + class Executable: """ @@ -107,9 +146,9 @@ def __call__( timeout: Optional[int] = ..., env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., - input: Optional[TextIO] = ..., - output: Union[Optional[TextIO], str] = ..., - error: Union[Optional[TextIO], str] = ..., + input: Optional[BinaryIO] = ..., + output: Union[Optional[BinaryIO], str] = ..., + error: Union[Optional[BinaryIO], str] = ..., _dump_env: Optional[Dict[str, str]] = ..., ) -> None: ... @@ -123,9 +162,9 @@ def __call__( timeout: Optional[int] = ..., env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., - input: Optional[TextIO] = ..., - output: Union[Type[str], Callable], - error: Union[Optional[TextIO], str, Type[str], Callable] = ..., + input: Optional[BinaryIO] = ..., + output: Union[Type[str], Callable], # str or str.split + error: OutType = ..., _dump_env: Optional[Dict[str, str]] = ..., ) -> str: ... @@ -139,9 +178,9 @@ def __call__( timeout: Optional[int] = ..., env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., - input: Optional[TextIO] = ..., - output: Union[Optional[TextIO], str, Type[str], Callable] = ..., - error: Union[Type[str], Callable], + input: Optional[BinaryIO] = ..., + output: OutType = ..., + error: Union[Type[str], Callable], # str or str.split _dump_env: Optional[Dict[str, str]] = ..., ) -> str: ... @@ -154,9 +193,9 @@ def __call__( timeout: Optional[int] = None, env: Optional[Union[Dict[str, str], EnvironmentModifications]] = None, extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = None, - input: Optional[TextIO] = None, - output: Union[Optional[TextIO], str, Type[str], Callable] = None, - error: Union[Optional[TextIO], str, Type[str], Callable] = None, + input: Optional[BinaryIO] = None, + output: OutType = None, + error: OutType = None, _dump_env: Optional[Dict[str, str]] = None, ) -> Optional[str]: """Runs this executable in a subprocess. @@ -195,29 +234,6 @@ def __call__( By default, the subprocess inherits the parent's file descriptors. """ - - def process_cmd_output(out, err): - result = None - if output in (str, str.split) or error in (str, str.split): - result = "" - if output in (str, str.split): - if sys.platform == "win32": - outstr = str(out.decode("ISO-8859-1")) - else: - outstr = str(out.decode("utf-8")) - result += outstr - if output is str.split: - sys.stdout.write(outstr) - if error in (str, str.split): - if sys.platform == "win32": - errstr = str(err.decode("ISO-8859-1")) - else: - errstr = str(err.decode("utf-8")) - result += errstr - if error is str.split: - sys.stderr.write(errstr) - return result - # Setup default environment current_environment = os.environ.copy() if env is None else {} self._default_envmod.apply_modifications(current_environment) @@ -246,37 +262,31 @@ def process_cmd_output(out, err): if isinstance(ignore_errors, int): ignore_errors = (ignore_errors,) - if input is str: - raise ValueError("Cannot use `str` as input stream.") + if input is str or input is str.split: + raise ValueError("Cannot use `str` or `str.split` as input stream.") + elif isinstance(input, str): + istream, close_istream = open(input, "rb"), True + else: + istream, close_istream = input, False - def streamify(arg, mode): - if isinstance(arg, str): - return open(arg, mode), True # pylint: disable=unspecified-encoding - elif arg in (str, str.split): - return subprocess.PIPE, False - else: - return arg, False - - ostream, close_ostream = streamify(output, "wb") - estream, close_estream = streamify(error, "wb") - istream, close_istream = streamify(input, "rb") + ostream, close_ostream = _streamify_output(output, "output") + estream, close_estream = _streamify_output(error, "error") if not ignore_quotes: quoted_args = [arg for arg in args if re.search(r'^".*"$|^\'.*\'$', arg)] if quoted_args: tty.warn( - "Quotes in command arguments can confuse scripts like" " configure.", + "Quotes in command arguments can confuse scripts like configure.", "The following arguments may cause problems when executed:", str("\n".join([" " + arg for arg in quoted_args])), "Quotes aren't needed because spack doesn't use a shell. " "Consider removing them.", - "If multiple levels of quotation are required, use " "`ignore_quotes=True`.", + "If multiple levels of quotation are required, use `ignore_quotes=True`.", ) cmd = self.exe + list(args) - escaped_cmd = ["'%s'" % arg.replace("'", "'\"'\"'") for arg in cmd] - cmd_line_string = " ".join(escaped_cmd) + cmd_line_string = " ".join(shlex.quote(arg) for arg in cmd) tty.debug(cmd_line_string) result = None @@ -289,9 +299,16 @@ def streamify(arg, mode): env=current_environment, close_fds=False, ) - out, err = proc.communicate(timeout=timeout) + except OSError as e: + message = "Command: " + cmd_line_string + if " " in self.exe[0]: + message += "\nDid you mean to add a space to the command?" - result = process_cmd_output(out, err) + raise ProcessError(f"{self.exe[0]}: {e.strerror}", message) + + try: + out, err = proc.communicate(timeout=timeout) + result = _process_cmd_output(out, err, output, error) rc = self.returncode = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): long_msg = cmd_line_string @@ -302,18 +319,12 @@ def streamify(arg, mode): # stdout/stderr (e.g. if 'output' is not specified) long_msg += "\n" + result - raise ProcessError("Command exited with status %d:" % proc.returncode, long_msg) - except OSError as e: - message = "Command: " + cmd_line_string - if " " in self.exe[0]: - message += "\nDid you mean to add a space to the command?" - - raise ProcessError("%s: %s" % (self.exe[0], e.strerror), message) + raise ProcessError(f"Command exited with status {proc.returncode}:", long_msg) except subprocess.TimeoutExpired as te: proc.kill() out, err = proc.communicate() - result = process_cmd_output(out, err) + result = _process_cmd_output(out, err, output, error) long_msg = cmd_line_string + f"\n{result}" if fail_on_error: raise ProcessTimeoutError( @@ -324,11 +335,12 @@ def streamify(arg, mode): ) from te finally: - if close_ostream: + # The isinstance checks are only needed for type checking. + if close_ostream and isinstance(ostream, io.IOBase): ostream.close() - if close_estream: + if close_estream and isinstance(estream, io.IOBase): estream.close() - if close_istream: + if close_istream and isinstance(istream, io.IOBase): istream.close() return result @@ -336,14 +348,11 @@ def streamify(arg, mode): def __eq__(self, other): return hasattr(other, "exe") and self.exe == other.exe - def __neq__(self, other): - return not (self == other) - def __hash__(self): return hash((type(self),) + tuple(self.exe)) def __repr__(self): - return "" % self.exe + return f"" def __str__(self): return " ".join(self.exe) diff --git a/lib/spack/spack/util/file_cache.py b/lib/spack/spack/util/file_cache.py index 55626da053e5e5..4c2e4bd960c7be 100644 --- a/lib/spack/spack/util/file_cache.py +++ b/lib/spack/spack/util/file_cache.py @@ -2,27 +2,46 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import errno -import math +import hashlib import os import pathlib import shutil -from typing import IO, Dict, Optional, Tuple, Union +import tempfile +from contextlib import contextmanager +from typing import IO, Dict, Iterator, Optional, Tuple, Union from spack.error import SpackError from spack.llnl.util.filesystem import rename -from spack.util.lock import Lock, ReadTransaction, WriteTransaction +from spack.util.lock import Lock def _maybe_open(path: Union[str, pathlib.Path]) -> Optional[IO[str]]: try: return open(path, "r", encoding="utf-8") - except OSError as e: - if e.errno != errno.ENOENT: - raise + except IsADirectoryError: + raise CacheError("Cache file is not a file: %s" % path) + except PermissionError: + raise CacheError("Cannot access cache file: %s" % path) + except FileNotFoundError: return None +def _open_temp(context_dir: Union[str, pathlib.Path]) -> Tuple[IO[str], str]: + """Open a temporary file in a directory + + This implementation minimizes the number of system calls for the case + the target directory already exists. + """ + try: + fd, path = tempfile.mkstemp(dir=context_dir) + except FileNotFoundError: + os.makedirs(context_dir, exist_ok=True) + fd, path = tempfile.mkstemp(dir=context_dir) + + stream = os.fdopen(fd, "w", encoding="utf-8") + return stream, path + + class ReadContextManager: def __init__(self, path: Union[str, pathlib.Path]) -> None: self.path = path @@ -38,14 +57,18 @@ def __exit__(self, type, value, traceback): class WriteContextManager: - def __init__(self, path: str) -> None: + def __init__(self, path: Union[str, pathlib.Path]) -> None: self.path = path - self.tmp_path = f"{self.path}.tmp" def __enter__(self) -> Tuple[Optional[IO[str]], IO[str]]: """Return (old_file, new_file) file objects, where old_file is optional.""" - self.old_file = _maybe_open(self.path) - self.new_file = open(self.tmp_path, "w", encoding="utf-8") + try: + self.old_file = _maybe_open(self.path) + self.new_file, self.tmp_path = _open_temp(os.path.dirname(self.path)) + except PermissionError: + if self.old_file: + self.old_file.close() + raise CacheError(f"Insufficient permissions to write to file cache at {self.path}") return self.old_file, self.new_file def __exit__(self, type, value, traceback): @@ -54,7 +77,10 @@ def __exit__(self, type, value, traceback): self.new_file.close() if value: - os.remove(self.tmp_path) + try: + os.remove(self.tmp_path) + except OSError: + pass else: rename(self.tmp_path, self.path) @@ -87,7 +113,8 @@ def __init__(self, root: Union[str, pathlib.Path], timeout=120): self.root = root self.root.mkdir(parents=True, exist_ok=True) - self._locks: Dict[Union[pathlib.Path, str], Lock] = {} + self.lock_path = self.root / ".lock" + self._locks: Dict[str, Lock] = {} self.lock_timeout = timeout def destroy(self): @@ -102,101 +129,81 @@ def cache_path(self, key: Union[str, pathlib.Path]): """Path to the file in the cache for a particular key.""" return self.root / key - def _lock_path(self, key: Union[str, pathlib.Path]): - """Path to the file in the cache for a particular key.""" - keyfile = os.path.basename(key) - keydir = os.path.dirname(key) - - return self.root / keydir / ("." + keyfile + ".lock") + def _get_lock_offsets(self, key: str) -> Tuple[int, int]: + """Hash function to determine byte-range offsets for a key. Returns (start, length) for + the lock.""" + hasher = hashlib.sha256(key.encode("utf-8")) + hash_int = int.from_bytes(hasher.digest()[:8], "little") + start_offset = hash_int % (2**63 - 1) + return start_offset, 1 def _get_lock(self, key: Union[str, pathlib.Path]): - """Create a lock for a key, if necessary, and return a lock object.""" - if key not in self._locks: - self._locks[key] = Lock(str(self._lock_path(key)), default_timeout=self.lock_timeout) - return self._locks[key] - - def init_entry(self, key: Union[str, pathlib.Path]): - """Ensure we can access a cache file. Create a lock for it if needed. - - Return whether the cache file exists yet or not. - """ - cache_path = self.cache_path(key) - # Avoid using pathlib here to allow the logic below to - # function as is - # TODO: Maybe refactor the following logic for pathlib - exists = os.path.exists(cache_path) - if exists: - if not cache_path.is_file(): - raise CacheError("Cache file is not a file: %s" % cache_path) - - if not os.access(cache_path, os.R_OK): - raise CacheError("Cannot access cache file: %s" % cache_path) - else: - # if the file is hierarchical, make parent directories - parent = cache_path.parent - if parent != self.root: - parent.mkdir(parents=True, exist_ok=True) - - if not os.access(parent, os.R_OK | os.W_OK): - raise CacheError("Cannot access cache directory: %s" % parent) - - # ensure lock is created for this key - self._get_lock(key) - return exists - - def read_transaction(self, key: Union[str, pathlib.Path]): + """Create a lock for a key using byte-range offsets.""" + key_str = str(key) + + if key_str not in self._locks: + start, length = self._get_lock_offsets(key_str) + self._locks[key_str] = Lock( + str(self.lock_path), + start=start, + length=length, + default_timeout=self.lock_timeout, + desc=f"key:{key_str}", + ) + return self._locks[key_str] + + @contextmanager + def read_transaction(self, key: Union[str, pathlib.Path]) -> Iterator[Optional[IO[str]]]: """Get a read transaction on a file cache item. - Returns a ReadTransaction context manager and opens the cache file for - reading. You can use it like this:: + Returns a context manager that yields an open file object for reading, + or None if the cache file does not exist. You can use it like this:: with file_cache_object.read_transaction(key) as cache_file: - cache_file.read() + if cache_file is not None: + cache_file.read() """ - path = self.cache_path(key) - return ReadTransaction( - self._get_lock(key), acquire=lambda: ReadContextManager(path) # type: ignore - ) + lock = self._get_lock(key) + lock.acquire_read() + try: + with ReadContextManager(self.cache_path(key)) as f: + yield f + finally: + lock.release_read() - def write_transaction(self, key: Union[str, pathlib.Path]): + @contextmanager + def write_transaction( + self, key: Union[str, pathlib.Path] + ) -> Iterator[Tuple[Optional[IO[str]], IO[str]]]: """Get a write transaction on a file cache item. - Returns a WriteTransaction context manager that opens a temporary file - for writing. Once the context manager finishes, if nothing went wrong, - moves the file into place on top of the old file atomically. + Returns a context manager that yields (old_file, new_file) where old_file + is the existing cache file (or None), and new_file is a writable temporary + file. Once the context manager exits cleanly, moves the temporary file + into place atomically. """ path = self.cache_path(key) - if os.path.exists(path) and not os.access(path, os.W_OK): + lock = self._get_lock(key) + try: + lock.acquire_write() + except PermissionError: raise CacheError(f"Insufficient permissions to write to file cache at {path}") - - return WriteTransaction( - self._get_lock(key), acquire=lambda: WriteContextManager(path) # type: ignore - ) - - def mtime(self, key: Union[str, pathlib.Path]) -> float: - """Return modification time of cache file, or -inf if it does not exist. - - Time is in units returned by os.stat in the mtime field, which is - platform-dependent. - - """ - if not self.init_entry(key): - return -math.inf - else: - return self.cache_path(key).stat().st_mtime + try: + with WriteContextManager(str(path)) as (old, new): + yield old, new + finally: + lock.release_write() def remove(self, key: Union[str, pathlib.Path]): file = self.cache_path(key) lock = self._get_lock(key) + lock.acquire_write() try: - lock.acquire_write() file.unlink() - except OSError as e: - # File not found is OK, so remove is idempotent. - if e.errno != errno.ENOENT: - raise + except FileNotFoundError: + pass finally: lock.release_write() diff --git a/lib/spack/spack/util/git.py b/lib/spack/spack/util/git.py index 23510670113b39..75308b5076bcce 100644 --- a/lib/spack/spack/util/git.py +++ b/lib/spack/spack/util/git.py @@ -14,6 +14,7 @@ import spack.llnl.util.filesystem as fs import spack.llnl.util.lang import spack.util.executable as exe +from spack.util.environment import EnvironmentModifications # regex for a commit version COMMIT_VERSION = re.compile(r"^[a-f0-9]{40}$") @@ -78,6 +79,7 @@ def __call__(self, exe_version, value=None) -> List: # git@1.8.5 is when branch could also accept tag so we don't have to track ref types as closely # This also corresponds to system git on RHEL7 MIN_OPT_VERSION = (1, 8, 5, 2) +MIN_DIRECT_COMMIT_FETCH = (2, 5, 0) # Technically the flags existed earlier but we are pruning our logic to 1.8.5 or greater BRANCH = VersionConditionalOption("--branch", min_version=MIN_OPT_VERSION) @@ -102,7 +104,16 @@ def git(required: bool = ...) -> Optional[GitExecutable]: ... def git(required: bool = False) -> Optional[GitExecutable]: - """Get a git executable. Raises CommandNotFoundError if ``required`` and git is not found.""" + """Get a git executable. + + The returned executable automatically unsets ``GIT_EXTERNAL_DIFF`` and ``GIT_DIFF_OPTS`` + environment variables that can interfere with spack git diff operations. + + Args: + required (bool): if True, raises CommandNotFoundError when git is not found + + Returns: GitExecutable, or None if git is not found and required is False + """ git_path = _find_git() if not git_path: @@ -114,9 +125,16 @@ def git(required: bool = False) -> Optional[GitExecutable]: # If we're running under pytest, add this to ignore the fix for CVE-2022-39253 in # git 2.38.1+. Do this in one place; we need git to do this in all parts of Spack. - if git and "pytest" in sys.modules: + if "pytest" in sys.modules: git.add_default_arg("-c", "protocol.file.allow=always") + # Block environment variables that can interfere with git diff operations + # this can cause problems for spack ci verify-versions and spack repo show-version-updates + env_blocklist = EnvironmentModifications() + env_blocklist.unset("GIT_EXTERNAL_DIFF") + env_blocklist.unset("GIT_DIFF_OPTS") + git.add_default_envmod(env_blocklist) + return git @@ -202,7 +220,7 @@ def pull_checkout_branch( raise ValueError("depth must be a positive integer") fetch_args.append(f"--depth={depth}") - git_exe("fetch", *fetch_args, remote, f"{branch}:refs/remotes/{remote}/{branch}") + git_exe("fetch", *fetch_args, remote, f"refs/heads/{branch}:refs/remotes/{remote}/{branch}") git_exe("checkout", "--quiet", branch) try: @@ -308,10 +326,11 @@ def git_init_fetch(url, ref, depth=None, debug=False, dest=None, git_exe=None): # minimum criteria for fetching a single commit, but also requires server to be configured # fall-back to a process error so an old git version or a fetch failure from an nonsupporting # server can be caught the same way. - if ref and is_git_commit_sha(ref) and version < (2, 5, 0): + if ref and is_git_commit_sha(ref) and version < MIN_DIRECT_COMMIT_FETCH: raise exe.ProcessError("Git older than 2.5 detected, can't fetch commit directly") init = ["init"] remote = ["remote", "add", "origin", url] + config = ["config", "remote.origin.fetch", "+refs/heads/*:origin/refs/*"] fetch = ["fetch"] if not debug: @@ -319,8 +338,16 @@ def git_init_fetch(url, ref, depth=None, debug=False, dest=None, git_exe=None): if depth and protocol_supports_shallow_clone(url): fetch.extend(DEPTH(version, str(depth))) - fetch.extend([*FILTER_BLOB_NONE(version), url, ref]) - cmds = [init, remote, fetch] + filter_args = FILTER_BLOB_NONE(version) + if filter_args: + fetch.extend(filter_args) + fetch.extend([url, ref]) + + partial_clone = ["config", "extensions.partialClone", "true"] if filter_args else None + if partial_clone is not None: + cmds = [init, partial_clone, remote, config, fetch] + else: + cmds = [init, remote, config, fetch] _exec_git_commands_unique_dir(git_exe, cmds, debug, dest) diff --git a/lib/spack/spack/util/lock.py b/lib/spack/spack/util/lock.py index e407f7099cbd67..39a6bdaeef7351 100644 --- a/lib/spack/spack/util/lock.py +++ b/lib/spack/spack/util/lock.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Wrapper for ``spack.llnl.util.lock`` allows locking to be enabled/disabled.""" + import os import stat import sys @@ -48,11 +49,20 @@ def __init__( desc=desc, ) + def _reaffirm_lock(self) -> None: + if self._enable: + super()._reaffirm_lock() + def _lock(self, op: int, timeout: Optional[float] = 0.0) -> Tuple[float, int]: if self._enable: return super()._lock(op, timeout) return 0.0, 0 + def _poll_lock(self, op: int) -> bool: + if self._enable: + return super()._poll_lock(op) + return True + def _unlock(self) -> None: """Unlock call that always succeeds.""" if self._enable: diff --git a/lib/spack/spack/util/log_parse.py b/lib/spack/spack/util/log_parse.py index e27740ee741e21..91218513e59ca4 100644 --- a/lib/spack/spack/util/log_parse.py +++ b/lib/spack/spack/util/log_parse.py @@ -3,119 +3,93 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import io -import shutil -import sys -from typing import Optional, TextIO, Union +from typing import List, Optional, TextIO, Tuple, Union from spack.llnl.util.tty.color import cescape, colorize -from spack.util.ctest_log_parser import BuildError, BuildWarning, CTestLogParser +from spack.util.ctest_log_parser import BuildError, BuildWarning, CTestLogParser, LogEvent __all__ = ["parse_log_events", "make_log_context"] +_PARSER: Optional[CTestLogParser] = None + + def parse_log_events( - stream: Union[str, TextIO], context: int = 6, jobs: Optional[int] = None, profile: bool = False -): - """Extract interesting events from a log file as a list of LogEvent. + stream: Union[str, TextIO], context: int = 6, profile: bool = False, tail: int = 0 +) -> Tuple[List[BuildError], List[BuildWarning], Union[LogEvent, None]]: + """Extract interesting events from a log file. Args: stream: build log name or file object context: lines of context to extract around each log event - jobs: number of jobs to parse with; default ncpus profile: print out profile information for parsing + tail: if > 0, also return the last ``tail`` lines Returns: two lists containing :class:`~spack.util.ctest_log_parser.BuildError` and - :class:`~spack.util.ctest_log_parser.BuildWarning` objects. - - This is a wrapper around :class:`~spack.util.ctest_log_parser.CTestLogParser` that - lazily constructs a single ``CTestLogParser`` object. This ensures - that all the regex compilation is only done once. + :class:`~spack.util.ctest_log_parser.BuildWarning` objects, plus an optional + :class:`~spack.util.ctest_log_parser.LogEvent` for the tail (None when ``tail=0``). """ - parser = getattr(parse_log_events, "ctest_parser", None) - if parser is None: - parser = CTestLogParser(profile=profile) - setattr(parse_log_events, "ctest_parser", parser) - - result = parser.parse(stream, context, jobs) + global _PARSER + if profile: + parser = CTestLogParser(profile=True) + elif _PARSER is None: + _PARSER = parser = CTestLogParser() + else: + parser = _PARSER + result = parser.parse(stream, context, tail) if profile: parser.print_timings() return result -#: lazily constructed CTest log parser -parse_log_events.ctest_parser = None # type: ignore[attr-defined] - - -def _wrap(text, width): - """Break text into lines of specific width.""" - lines = [] - pos = 0 - while pos < len(text): - lines.append(text[pos : pos + width]) - pos += width - return lines - - -def make_log_context(log_events, width=None): +def make_log_context(log_events: List[LogEvent]) -> str: """Get error context from a log file. Args: - log_events (list): list of events created by - ``ctest_log_parser.parse()`` - width (int or None): wrap width; ``0`` for no limit; ``None`` to - auto-size for terminal + log_events: list of events created by ``ctest_log_parser.parse()`` + Returns: str: context from the build log with errors highlighted - Parses the log file for lines containing errors, and prints them out - with line numbers and context. Errors are highlighted with ``>>`` and - with red highlighting (if color is enabled). - - Events are sorted by line number before they are displayed. + Parses the log file for lines containing errors, and prints them out with context. + Errors are highlighted in red and warnings in yellow. Events are sorted by line number. """ - error_lines = set(e.line_no for e in log_events) + event_colors = {e.line_no: e.color for e in log_events if e.color} log_events = sorted(log_events, key=lambda e: e.line_no) - num_width = len(str(max(error_lines or [0]))) + 4 - line_fmt = "%%-%dd%%s" % num_width - indent = " " * (5 + num_width) - - if width is None: - width = shutil.get_terminal_size().columns - if width <= 0: - width = sys.maxsize - wrap_width = width - num_width - 6 - out = io.StringIO() next_line = 1 - for event in log_events: - start = event.start + block_start = -1 + block_lines: List[str] = [] - if isinstance(event, BuildError): - color = "R" - elif isinstance(event, BuildWarning): - color = "Y" - else: - color = "W" + def flush_block(): + block_end = block_start + len(block_lines) - 1 + out.write(colorize("@c{-- lines %d to %d --}\n" % (block_start, block_end))) + out.writelines(block_lines) + block_lines.clear() - if next_line != 1 and start > next_line: - out.write("\n ...\n\n") + for event in log_events: + start = event.start if start < next_line: start = next_line + elif block_lines: + flush_block() - for i in range(start, event.end): - # wrap to width - lines = _wrap(event[i], wrap_width) - lines[1:] = [indent + ln for ln in lines[1:]] - wrapped_line = line_fmt % (i, "\n".join(lines)) + if not block_lines: + block_start = start - if i in error_lines: - out.write(colorize(" @%s{>> %s}\n" % (color, cescape(wrapped_line)))) + for i in range(start, event.end): + if i in event_colors: + color = event_colors[i] + block_lines.append(colorize("@%s{> %s}\n" % (color, cescape(event[i])))) else: - out.write(" %s\n" % wrapped_line) + block_lines.append(" %s\n" % event[i]) next_line = event.end + if block_lines: + flush_block() + return out.getvalue() diff --git a/lib/spack/spack/util/module_cmd.py b/lib/spack/spack/util/module_cmd.py index a6cb1c8cd3daa9..3294b1bb053916 100644 --- a/lib/spack/spack/util/module_cmd.py +++ b/lib/spack/spack/util/module_cmd.py @@ -6,12 +6,14 @@ This module contains routines related to the module command for accessing and parsing environment modules. """ + import os import re import subprocess from typing import MutableMapping, Optional import spack.llnl.util.tty as tty +from spack.error import SpackError # This list is not exhaustive. Currently we only use load and unload # If we need another option that changes the environment, add it here. @@ -87,6 +89,9 @@ def load_module(mod): """Takes a module name and removes modules until it is possible to load that module. It then loads the provided module. Depends on the modulecmd implementation of modules used in cray and lmod. + + Raises: + ModuleLoadError: if the module could not be loaded """ tty.debug("module_cmd.load_module: {0}".format(mod)) # Read the module and remove any conflicting modules @@ -98,10 +103,20 @@ def load_module(mod): if word == "conflict": module("unload", text[i + 1]) + # Store the LOADEDMODULES before trying to load the new module + loaded_modules_before = os.environ.get("LOADEDMODULES", "") + # Load the module now that there are no conflicts # Some module systems use stdout and some use stderr module("load", mod) + # Check if the module was actually loaded by comparing LOADEDMODULES + loaded_modules_after = os.environ.get("LOADEDMODULES", "") + + # If LOADEDMODULES didn't change, the module wasn't loaded + if loaded_modules_before == loaded_modules_after: + raise ModuleLoadError(mod) + def get_path_args_from_module_line(line): if "(" in line and ")" in line: @@ -148,7 +163,7 @@ def path_from_modules(modules): candidate_path = get_path_from_module_contents(text, module_name) if candidate_path and not os.path.exists(candidate_path): - msg = "Extracted path from module does not exist " "[module={0}, path={1}]" + msg = "Extracted path from module does not exist [module={0}, path={1}]" tty.warn(msg.format(module_name, candidate_path)) # If anything is found, then it's the best choice. This means @@ -188,7 +203,7 @@ def match_pattern_and_strip(line, pattern, strip=[]): def match_flag_and_strip(line, flag, strip=[]): flag_idx = line.find(flag) if flag_idx >= 0: - # Search for the first occurence of any separator marking the end of + # Search for the first occurrence of any separator marking the end of # the path. separators = (" ", '"', "'") occurrences = [line.find(s, flag_idx) for s in separators] @@ -237,3 +252,10 @@ def match_flag_and_strip(line, flag, strip=[]): # Unable to find path in module return None + + +class ModuleLoadError(SpackError): + """Raised when a module cannot be loaded.""" + + def __init__(self, module): + super().__init__(f"Module '{module}' could not be loaded.") diff --git a/lib/spack/spack/util/package_hash.py b/lib/spack/spack/util/package_hash.py index 563843ae9f1369..806e08cede62f3 100644 --- a/lib/spack/spack/util/package_hash.py +++ b/lib/spack/spack/util/package_hash.py @@ -96,7 +96,7 @@ def visit_Expr(self, node): # opposed to function calls through a variable callback). We remove them. # # Note that changes to directives (e.g., a preferred version change or a hash - # chnage on an archive) are already represented in the spec *outside* the + # change on an archive) are already represented in the spec *outside* the # package hash. return ( None @@ -157,7 +157,7 @@ def visit_ClassDef(self, node): if self.in_classdef: return node - # guard against recrusive class definitions + # guard against recursive class definitions self.in_classdef = True self.generic_visit(node) self.in_classdef = False diff --git a/lib/spack/spack/util/parallel.py b/lib/spack/spack/util/parallel.py index ae05905f062f48..ef60753b44141d 100644 --- a/lib/spack/spack/util/parallel.py +++ b/lib/spack/spack/util/parallel.py @@ -60,7 +60,13 @@ def __call__(self, *args, **kwargs): def imap_unordered( - f, list_of_args, *, processes: int, maxtaskperchild: Optional[int] = None, debug=False + f, + list_of_args, + *, + processes: int, + maxtaskperchild: Optional[int] = None, + debug=False, + serialize_env: bool = False, ): """Wrapper around multiprocessing.Pool.imap_unordered. @@ -83,7 +89,7 @@ def imap_unordered( from spack.subprocess_context import GlobalStateMarshaler - marshaler = GlobalStateMarshaler() + marshaler = GlobalStateMarshaler(serialize_env=serialize_env) with multiprocessing.Pool( processes, initializer=marshaler.restore, maxtasksperchild=maxtaskperchild ) as p: @@ -107,22 +113,18 @@ def submit(self, fn, *args, **kwargs): def make_concurrent_executor( - jobs: Optional[int] = None, *, require_fork: bool = True + jobs: Optional[int] = None, *, serialize_env: bool = False ) -> concurrent.futures.Executor: - """Create a concurrent executor. If require_fork is True, then the executor is sequential - if the platform does not enable forking as the default start method. Effectively - require_fork=True makes the executor sequential in the current process on Windows, macOS, and - Linux from Python 3.14+ (which changes defaults)""" - - if ( - not ENABLE_PARALLELISM - or (require_fork and multiprocessing.get_start_method() != "fork") - or sys.version_info[:2] == (3, 6) - ): + """Create a concurrent executor. + + If serialize_env is False (default), the active Spack environment is not transmitted to the + worker processes, which avoids the cost of pickling potentially large environment state.""" + + if not ENABLE_PARALLELISM or sys.version_info[:2] == (3, 6): return SequentialExecutor() from spack.subprocess_context import GlobalStateMarshaler jobs = jobs or spack.config.determine_number_of_jobs(parallel=True) - marshaler = GlobalStateMarshaler() + marshaler = GlobalStateMarshaler(serialize_env=serialize_env) return concurrent.futures.ProcessPoolExecutor(jobs, initializer=marshaler.restore) # novermin diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index afa7936ba27a93..bbd6a27f506dac 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -6,6 +6,7 @@ TODO: this is really part of spack.config. Consolidate it. """ + import contextlib import getpass import os @@ -15,7 +16,7 @@ import sys import tempfile from datetime import date -from typing import Optional +from typing import Optional, Union import spack.llnl.util.tty as tty import spack.util.spack_yaml as syaml @@ -96,9 +97,12 @@ def replacements(): #: Padded paths comprise directories with this name (or some prefix of it). : #: It starts with two underscores to make it unlikely that prefix matches would -#: include some other component of the intallation path. +#: include some other component of the installation path. SPACK_PATH_PADDING_CHARS = "__spack_path_placeholder__" +#: Bytes equivalent of SPACK_PATH_PADDING_CHARS. +SPACK_PATH_PADDING_BYTES = SPACK_PATH_PADDING_CHARS.encode("ascii") + #: Special padding char if the padded string would otherwise end with a path #: separator (since the path separator would otherwise get collapsed out, #: causing inconsistent padding). @@ -318,12 +322,29 @@ def longest_prefix_re(string, capture=True): ) -#: regex cache for padding_filter function -_filter_re = None +def _build_padding_re(as_bytes: bool = False): + """Build and return a compiled regex for filtering path padding placeholders.""" + pad = re.escape(SPACK_PATH_PADDING_CHARS) + extra = SPACK_PATH_PADDING_EXTRA_CHAR + longest_prefix = longest_prefix_re(SPACK_PATH_PADDING_CHARS, capture=False) + regex = ( + r"((?:/[^/\s]*)*?)" # zero or more leading non-whitespace path components + r"(?:/{pad})+" # the padding string repeated one or more times + # trailing prefix of padding as path component + r"(?:/{longest_prefix}|/{longest_prefix}{extra})?(?=/)" + ) + regex = regex.replace("/", re.escape(os.sep)) + regex = regex.format(pad=pad, extra=extra, longest_prefix=longest_prefix) -def padding_filter(string): - """Filter used to reduce output from path padding in log output. + if as_bytes: + return re.compile(regex.encode("ascii")) + else: + return re.compile(regex) + + +class _PaddingFilter: + """Callable that filters path-padding placeholders from a string or bytes buffer. This turns paths like this: @@ -335,7 +356,7 @@ def padding_filter(string): Where ``padded-to-512-chars`` indicates that the prefix was padded with placeholders until it hit 512 characters. The actual value of this number - depends on what the `install_tree``'s ``padded_length`` is configured to. + depends on what the ``install_tree``'s ``padded_length`` is configured to. For a path to match and be filtered, the placeholder must appear in its entirety at least one time. e.g., "/spack/" would not be filtered, but @@ -343,29 +364,32 @@ def padding_filter(string): Note that only the first padded path in the string is filtered. """ - global _filter_re - - pad = SPACK_PATH_PADDING_CHARS - if not _filter_re: - longest_prefix = longest_prefix_re(pad) - regex = ( - r"((?:/[^/\s]*)*?)" # zero or more leading non-whitespace path components - r"(/{pad})+" # the padding string repeated one or more times - # trailing prefix of padding as path component - r"(/{longest_prefix}|/{longest_prefix}{extra_pad_character})?(?=/)" - ) - regex = regex.replace("/", re.escape(os.sep)) - regex = regex.format( - pad=pad, - extra_pad_character=SPACK_PATH_PADDING_EXTRA_CHAR, - longest_prefix=longest_prefix, - ) - _filter_re = re.compile(regex) - - def replacer(match): - return "%s%s[padded-to-%d-chars]" % (match.group(1), os.sep, len(match.group(0))) - - return _filter_re.sub(replacer, string) + + __slots__ = ("_re", "_needle", "_fmt") + + def __init__(self, as_bytes: bool = False) -> None: + self._re = _build_padding_re(as_bytes=as_bytes) + if as_bytes: + self._needle: Union[str, bytes] = SPACK_PATH_PADDING_BYTES + self._fmt: Union[str, bytes] = b"%b" + os.sep.encode("ascii") + b"[padded-to-%d-chars]" + else: + self._needle = SPACK_PATH_PADDING_CHARS + self._fmt = "%s" + os.sep + "[padded-to-%d-chars]" + + def _replace(self, match): + return self._fmt % (match.group(1), len(match.group(0))) + + def __call__(self, data): + if self._needle not in data: + return data + return self._re.sub(self._replace, data) + + +#: Callable that filters path-padding placeholders from strings +padding_filter = _PaddingFilter(as_bytes=False) + +#: Callable that filters path-padding placeholders from bytes buffers +padding_filter_bytes = _PaddingFilter(as_bytes=True) @contextlib.contextmanager @@ -380,7 +404,7 @@ def filter_padding(): padding = spack.config.get("config:install_tree:padded_length", None) if padding: - # filter out all padding from the intsall command output + # filter out all padding from the install command output with tty.output_filter(padding_filter): yield else: diff --git a/lib/spack/spack/util/prefix.py b/lib/spack/spack/util/prefix.py index 09184080720791..6d6543eaad1d3c 100644 --- a/lib/spack/spack/util/prefix.py +++ b/lib/spack/spack/util/prefix.py @@ -5,6 +5,7 @@ """ This file contains utilities for managing the installation prefix of a package. """ + import os from typing import Dict diff --git a/lib/spack/spack/util/remote_file_cache.py b/lib/spack/spack/util/remote_file_cache.py index 0897e33ca20778..413a693cab0ac9 100644 --- a/lib/spack/spack/util/remote_file_cache.py +++ b/lib/spack/spack/util/remote_file_cache.py @@ -9,7 +9,7 @@ import tempfile import urllib.parse import urllib.request -from typing import Callable, Optional +from typing import Optional import spack.llnl.util.tty as tty import spack.util.crypto @@ -60,13 +60,13 @@ def fetch_remote_text_file(url: str, dest_dir: str) -> str: return fetch_url_text(raw_url, dest_dir=dest_dir) -def local_path(raw_path: str, sha256: str, make_dest: Optional[Callable[[], str]] = None) -> str: +def local_path(raw_path: str, sha256: str, dest: Optional[str] = None) -> str: """Determine the actual path and, if remote, stage its contents locally. Args: raw_path: raw path with possible variables needing substitution - sha256: the expected sha256 for the file - make_dest: function to create a stage for remote files, if needed (e.g., ``mkdtemp``) + sha256: the expected sha256 if the file is remote + dest: destination path Returns: resolved, normalized local path @@ -80,7 +80,7 @@ def local_path(raw_path: str, sha256: str, make_dest: Optional[Callable[[], str] # Allow paths (and URLs) to contain spack config/environment variables, # etc. - path = canonicalize_path(raw_path) + path = canonicalize_path(raw_path, dest) # Save off the Windows drive of the canonicalized path (since now absolute) # to ensure recognized by URL parsing as a valid file "scheme". @@ -98,8 +98,11 @@ def local_path(raw_path: str, sha256: str, make_dest: Optional[Callable[[], str] if validate_scheme(url.scheme): # Fetch files from supported URL schemes. if url.scheme in ("http", "https", "ftp"): - if make_dest is None: + if not dest: raise ValueError("Requires the destination argument to cache remote files") + assert os.path.isabs(dest), ( + f"Remote file destination '{dest}' must be an absolute path" + ) # Stage the remote configuration file tmpdir = tempfile.mkdtemp() @@ -118,7 +121,7 @@ def local_path(raw_path: str, sha256: str, make_dest: Optional[Callable[[], str] raise ValueError(f"Requires sha256 ('{checksum}') to cache remote files.") # Copy the file to the destination directory - dest_dir = join_path(make_dest(), checksum) + dest_dir = join_path(dest, checksum) if not os.path.exists(dest_dir): mkdirp(dest_dir) diff --git a/lib/spack/spack/util/spack_json.py b/lib/spack/spack/util/spack_json.py index 40bd816c353bf1..ad0e17a68fd4f9 100644 --- a/lib/spack/spack/util/spack_json.py +++ b/lib/spack/spack/util/spack_json.py @@ -3,14 +3,18 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Simple wrapper around JSON to guarantee consistent use of load/dump.""" + import json -from typing import Any, Dict, Optional +from typing import IO, Any, Dict import spack.error -__all__ = ["load", "dump", "SpackJSONError"] +__all__ = ["load", "dump", "dumps", "SpackJSONError"] -_json_dump_args = {"indent": None, "separators": (",", ":")} +_DEFAULT_SEPARATORS = (",", ":") +_DEFAULT_INDENT = None +_PRETTY_SEPARATORS = (", ", ": ") +_PRETTY_INDENT = " " def load(stream: Any) -> Dict: @@ -20,12 +24,18 @@ def load(stream: Any) -> Dict: return json.load(stream) -def dump(data: Dict, stream: Optional[Any] = None) -> Optional[str]: - """Dump JSON with a reasonable amount of indentation and separation.""" - if stream is None: - return json.dumps(data, **_json_dump_args) # type: ignore[arg-type] - json.dump(data, stream, **_json_dump_args) # type: ignore[arg-type] - return None +def dump(data: Any, stream: IO[str], pretty: bool = False) -> None: + """Wrapper around json.dump with different default arguments""" + indent = _PRETTY_INDENT if pretty else _DEFAULT_INDENT + separators = _PRETTY_SEPARATORS if pretty else _DEFAULT_SEPARATORS + json.dump(data, stream, separators=separators, indent=indent) + + +def dumps(data: Any, pretty: bool = False) -> str: + """Wrapper around json.dumps with different default arguments""" + indent = _PRETTY_INDENT if pretty else _DEFAULT_INDENT + separators = _PRETTY_SEPARATORS if pretty else _DEFAULT_SEPARATORS + return json.dumps(data, separators=separators, indent=indent) class SpackJSONError(spack.error.SpackError): diff --git a/lib/spack/spack/util/spack_yaml.py b/lib/spack/spack/util/spack_yaml.py index f1aa1dead2a779..88ff506d869fdd 100644 --- a/lib/spack/spack/util/spack_yaml.py +++ b/lib/spack/spack/util/spack_yaml.py @@ -8,9 +8,10 @@ us to access file and line information later. - ``Our load methods use ``OrderedDict`` class instead of YAML's - default unorderd dict. + default unordered dict. """ + import ctypes import enum import functools diff --git a/lib/spack/spack/util/timer.py b/lib/spack/spack/util/timer.py index b275c71b91b431..e01c60ec20356f 100644 --- a/lib/spack/spack/util/timer.py +++ b/lib/spack/spack/util/timer.py @@ -8,6 +8,7 @@ a stack trace and drops the user into an interpreter. """ + import collections import sys import time @@ -180,7 +181,7 @@ def write_json(self, out=sys.stdout, extra_attributes={}): if extra_attributes: data.update(extra_attributes) if out: - out.write(sjson.dump(data)) + out.write(sjson.dumps(data)) else: return data diff --git a/lib/spack/spack/util/typing.py b/lib/spack/spack/util/typing.py index 9c499e3577edce..53b0fbc33809bd 100644 --- a/lib/spack/spack/util/typing.py +++ b/lib/spack/spack/util/typing.py @@ -11,7 +11,6 @@ """ - from typing import TYPE_CHECKING, Any from spack.vendor.typing_extensions import Protocol diff --git a/lib/spack/spack/util/unparse/unparser.py b/lib/spack/spack/util/unparse/unparser.py index 7b5a2ab615ea0e..ba9fe8fd976e26 100644 --- a/lib/spack/spack/util/unparse/unparser.py +++ b/lib/spack/spack/util/unparse/unparser.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: Python-2.0 "Usage: unparse.py " + import ast import sys from ast import AST, FormattedValue, If, JoinedStr, Name, Tuple diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index 550cb5225690a9..d2bede6717df75 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -4,22 +4,28 @@ import email.message import errno +import functools import io import json import os import re import shutil +import socket import ssl import stat import sys +import time import traceback import urllib.parse from html.parser import HTMLParser +from http.client import IncompleteRead from pathlib import Path, PurePosixPath -from typing import IO, Dict, Iterable, List, Optional, Set, Tuple, Union +from typing import IO, Callable, Dict, Iterable, List, Optional, Set, Tuple, TypeVar, Union from urllib.error import HTTPError, URLError from urllib.request import HTTPDefaultErrorHandler, HTTPSHandler, Request, build_opener +from spack.vendor.typing_extensions import ParamSpec + import spack import spack.config import spack.error @@ -36,6 +42,52 @@ from .s3 import UrllibS3Handler, get_s3_session +def is_transient_error(e: Exception) -> bool: + """Return True for HTTP/network errors that are worth retrying.""" + + if isinstance(e, HTTPError) and (500 <= e.code < 600 or e.code == 429): + return True + if isinstance(e, URLError) and isinstance(e.reason, socket.timeout): + return True + if isinstance(e, (socket.timeout, IncompleteRead)): + return True + # exceptions not inherited from the above used in urllib3 and botocore. + if type(e).__name__ in ( + "ConnectionClosedError", + "IncompleteReadError", + "ProtocolError", + "ReadTimeoutError", + "ResponseStreamingError", + ): + return True + return False + + +_P = ParamSpec("_P") +_R = TypeVar("_R") + + +def retry_on_transient_error( + f: Callable[_P, _R], retries: int = 5, sleep: Optional[Callable[[float], None]] = None +) -> Callable[_P, _R]: + """Retry a function on transient HTTP/network errors with exponential backoff.""" + sleep = sleep or time.sleep + + @functools.wraps(f) + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: + for i in range(retries): + try: + return f(*args, **kwargs) + except Exception as e: + if i + 1 != retries and is_transient_error(e): + sleep(2**i) # type: ignore[misc] # mypy still thinks it's possibly None. + continue + raise + raise AssertionError("unreachable") + + return wrapper + + class DetailedHTTPError(HTTPError): def __init__( self, req: Request, code: int, msg: str, hdrs: email.message.Message, fp: Optional[IO] @@ -188,7 +240,7 @@ def handle_starttag(self, tag, attrs): # GitLab uses a javascript function to place dropdown links: #
    + # data-download-links="[{"path":"/graphviz/graphviz/-/archive/12.0.0/graphviz-12.0.0.zip",...},...]"/> # noqa: E501 if tag == "div" and ("class", "js-source-code-dropdown") in attrs: try: links_str = next(val for key, val in attrs if key == "data-download-links") @@ -252,6 +304,38 @@ def read_from_url(url, accept_content_type=None): return response.url, response.headers, response +def _read_text(url: str) -> str: + request = Request(url, headers={"User-Agent": SPACK_USER_AGENT}) + with urlopen(request) as response: + return io.TextIOWrapper(response, encoding="utf-8").read() + + +def _read_json(url: str): + request = Request(url, headers={"User-Agent": SPACK_USER_AGENT}) + with urlopen(request) as response: + return json.load(response) + + +_read_text_with_retry = retry_on_transient_error(_read_text) +_read_json_with_retry = retry_on_transient_error(_read_json) + + +def read_text(url: str) -> str: + """Fetch url and return the response body decoded as UTF-8 text.""" + try: + return _read_text_with_retry(url) + except Exception as e: + raise SpackWebError(f"Download of {url} failed: {e.__class__.__name__}: {e}") + + +def read_json(url: str): + """Fetch url and return the response body parsed as JSON.""" + try: + return _read_json_with_retry(url) + except Exception as e: + raise SpackWebError(f"Download of {url} failed: {e.__class__.__name__}: {e}") + + def push_to_url(local_file_path, remote_path, keep_original=True, extra_args=None): remote_url = urllib.parse.urlparse(remote_path) if remote_url.scheme == "file": @@ -411,7 +495,7 @@ def fetch_url_text(url, curl: Optional[Executable] = None, dest_dir="."): fetch_method = spack.config.get("config:url_fetch_method") tty.debug("Using '{0}' to fetch {1} into {2}".format(fetch_method, url, path)) - if fetch_method.startswith("curl"): + if fetch_method and fetch_method.startswith("curl"): curl_exe = curl or require_curl() curl_args = fetch_method.split()[1:] + ["-O"] curl_args.extend(base_curl_fetch_args(url)) @@ -425,9 +509,7 @@ def fetch_url_text(url, curl: Optional[Executable] = None, dest_dir="."): else: try: - _, _, response = read_from_url(url) - - output = io.TextIOWrapper(response, encoding="utf-8").read() + output = read_text(url) if output: with working_dir(dest_dir, create=True): with open(filename, "w", encoding="utf-8") as f: @@ -441,6 +523,17 @@ def fetch_url_text(url, curl: Optional[Executable] = None, dest_dir="."): return None +def _url_exists_urllib_impl(url): + with urlopen( + Request(url, method="HEAD", headers={"User-Agent": SPACK_USER_AGENT}), + timeout=spack.config.get("config:connect_timeout", 10), + ) as _: + pass + + +_url_exists_urllib = retry_on_transient_error(_url_exists_urllib_impl) + + def url_exists(url, curl=None): """Determines whether url exists. @@ -474,12 +567,9 @@ def url_exists(url, curl=None): # Otherwise use urllib. try: - urlopen( - Request(url, method="HEAD", headers={"User-Agent": SPACK_USER_AGENT}), - timeout=spack.config.get("config:connect_timeout", 10), - ) + _url_exists_urllib(url) return True - except OSError as e: + except Exception as e: tty.debug(f"Failure reading {url}: {e}") return False @@ -696,7 +786,7 @@ def spider( root = urllib.parse.urlparse(root_str) spider_args.append((root, go_deeper, _visited)) - with spack.util.parallel.make_concurrent_executor(concurrency, require_fork=False) as tp: + with spack.util.parallel.make_concurrent_executor(concurrency) as tp: while current_depth <= depth: tty.debug( f"SPIDER: [depth={current_depth}, max_depth={depth}, urls={len(spider_args)}]" @@ -745,7 +835,8 @@ def _spider(url: urllib.parse.ParseResult, collect_nested: bool, _visited: Set[s if not response_url or not response: return pages, links, subcalls, _visited - page = io.TextIOWrapper(response, encoding="utf-8").read() + with response: + page = io.TextIOWrapper(response, encoding="utf-8").read() pages[response_url] = page # Parse out the include-fragments in the page @@ -772,7 +863,8 @@ def _spider(url: urllib.parse.ParseResult, collect_nested: bool, _visited: Set[s if not fragment_response_url or not fragment_response: continue - fragment = io.TextIOWrapper(fragment_response, encoding="utf-8").read() + with fragment_response: + fragment = io.TextIOWrapper(fragment_response, encoding="utf-8").read() fragments.add(fragment) pages[fragment_response_url] = fragment diff --git a/lib/spack/spack/util/windows_registry.py b/lib/spack/spack/util/windows_registry.py index 12d2b3b2359ad7..e205cff5ffd991 100644 --- a/lib/spack/spack/util/windows_registry.py +++ b/lib/spack/spack/util/windows_registry.py @@ -408,7 +408,7 @@ def __init__(self, key): class InvalidRegistryOperation(RegistryError): - """A Runtime Error ecountered when a registry operation is invalid for + """A Runtime Error encountered when a registry operation is invalid for an indeterminate reason""" def __init__(self, name, e, *args, **kwargs): diff --git a/lib/spack/spack/variant.py b/lib/spack/spack/variant.py index 30a0581e3a5d15..35fe12a0085af9 100644 --- a/lib/spack/spack/variant.py +++ b/lib/spack/spack/variant.py @@ -5,6 +5,7 @@ """The variant module contains data structures that are needed to manage variants both in packages and in specs. """ + import collections.abc import enum import functools @@ -827,7 +828,7 @@ class InconsistentValidationError(spack.error.SpecError): """Raised if the wrong validator is used to validate a variant.""" def __init__(self, vspec, variant): - msg = 'trying to validate variant "{0.name}" ' 'with the validator of "{1.name}"' + msg = 'trying to validate variant "{0.name}" with the validator of "{1.name}"' super().__init__(msg.format(vspec, variant)) diff --git a/lib/spack/spack/verify.py b/lib/spack/spack/verify.py index 105d206e8562a6..33ff521d81e3aa 100644 --- a/lib/spack/spack/verify.py +++ b/lib/spack/spack/verify.py @@ -193,7 +193,7 @@ def has_errors(self): return bool(self.errors) def json_string(self): - return sjson.dump(self.errors) + return sjson.dumps(self.errors) def __str__(self): res = "" diff --git a/lib/spack/spack/verify_libraries.py b/lib/spack/spack/verify_libraries.py index c69c9de28b5453..c6e4c8c7f62682 100644 --- a/lib/spack/spack/verify_libraries.py +++ b/lib/spack/spack/verify_libraries.py @@ -110,7 +110,7 @@ def visit_file(self, root: str, rel_path: str, depth: int) -> None: # We work with byte strings for paths. path = os.path.join(root, rel_path).encode("utf-8") - # For $ORIGIN interpolation: should not have trailing dir seperator. + # For $ORIGIN interpolation: should not have trailing dir separator. origin = os.path.dirname(path) # Retrieve the needed libs + rpaths. diff --git a/lib/spack/spack/version/git_ref_lookup.py b/lib/spack/spack/version/git_ref_lookup.py index 607a97e3a16b7c..9a7b7653a80eed 100644 --- a/lib/spack/spack/version/git_ref_lookup.py +++ b/lib/spack/spack/version/git_ref_lookup.py @@ -60,9 +60,6 @@ def cache_key(self): key_base = "git_metadata" self._cache_key = (Path(key_base) / self.repository_uri).as_posix() - # Cache data in MISC_CACHE - # If this is the first lazy access, initialize the cache as well - spack.caches.MISC_CACHE.init_entry(self.cache_key) return self._cache_key @property @@ -103,8 +100,8 @@ def save(self): def load_data(self): """Load data if the path already exists.""" - if os.path.isfile(self.cache_path): - with spack.caches.MISC_CACHE.read_transaction(self.cache_key) as cache_file: + with spack.caches.MISC_CACHE.read_transaction(self.cache_key) as cache_file: + if cache_file is not None: self.data = sjson.load(cache_file) def get(self, ref) -> Tuple[Optional[str], int]: diff --git a/lib/spack/spack/version/version_types.py b/lib/spack/spack/version/version_types.py index aea3aaadbdc48e..b41d5707f199aa 100644 --- a/lib/spack/spack/version/version_types.py +++ b/lib/spack/spack/version/version_types.py @@ -131,7 +131,7 @@ def parse_string_components(string: str) -> Tuple[VersionTuple, SeparatorTuple]: raise ValueError("Bad characters in version string: %s" % string) segments = SEGMENT_REGEX.findall(string) - separators: Tuple[str] = tuple(m[2] for m in segments) + separators: Tuple[str] = tuple([m[2] for m in segments]) prerelease: Tuple[int, ...] # (alpha|beta|rc) @@ -149,7 +149,7 @@ def parse_string_components(string: str) -> Tuple[VersionTuple, SeparatorTuple]: prerelease = (FINAL,) release: VersionComponentTuple = tuple( - int(m[0]) if m[0] else VersionStrComponent.from_string(m[1]) for m in segments + [int(m[0]) if m[0] else VersionStrComponent.from_string(m[1]) for m in segments] ) return (release, prerelease), separators @@ -245,7 +245,8 @@ def __init__(self, string: str, version: VersionTuple, separators: Tuple[str, .. @staticmethod def from_string(string: str) -> "StandardVersion": - return StandardVersion(string, *parse_string_components(string)) + version, separators = parse_string_components(string) + return StandardVersion(string, version, separators) @staticmethod def typemin() -> "StandardVersion": @@ -365,18 +366,18 @@ def intersects(self, other: VersionType) -> bool: return other.intersects(self) def satisfies(self, other: VersionType) -> bool: + if isinstance(other, VersionList): + return other.intersects(self) + + if isinstance(other, ClosedOpenRange): + return other.intersects(self) + if isinstance(other, GitVersion): return False if isinstance(other, StandardVersion): return self == other - if isinstance(other, ClosedOpenRange): - return other.intersects(self) - - if isinstance(other, VersionList): - return other.intersects(self) - raise NotImplementedError def union(self, other: VersionType) -> VersionType: @@ -594,7 +595,7 @@ def ref_version(self) -> StandardVersion: if self.ref_lookup is None: raise VersionLookupError( - f"git ref '{self.ref}' cannot be looked up: " "call attach_lookup first" + f"git ref '{self.ref}' cannot be looked up: call attach_lookup first" ) version_string, distance = self.ref_lookup.get(self.ref) @@ -683,7 +684,7 @@ def __le__(self, other: object) -> bool: if isinstance(other, GitVersion): return (self.ref_version, self.ref) <= (other.ref_version, other.ref) if isinstance(other, StandardVersion): - # Note: GitVersion hash=1.2.3 > StandardVersion 1.2.3, so use < comparsion. + # Note: GitVersion hash=1.2.3 > StandardVersion 1.2.3, so use < comparison. return self.ref_version < other if isinstance(other, ClosedOpenRange): # Equality is not a thing @@ -773,45 +774,65 @@ def up_to(self, index) -> StandardVersion: return self.ref_version.up_to(index) +def _str_range(lo: StandardVersion, hi: StandardVersion) -> str: + """Create a string representation from lo:hi range.""" + if lo == _STANDARD_VERSION_TYPEMIN: + if hi == _STANDARD_VERSION_TYPEMAX: + return ":" + else: + return f":{hi}" + elif hi == _STANDARD_VERSION_TYPEMAX: + return f"{lo}:" + elif lo == hi: + return str(lo) + else: + return f"{lo}:{hi}" + + class ClosedOpenRange(VersionType): - __slots__ = ("lo", "hi") + __slots__ = ("lo", "hi", "_string", "_hash") def __init__(self, lo: StandardVersion, hi: StandardVersion): if hi < lo: raise EmptyRangeError(f"{lo}..{hi} is an empty range") self.lo: StandardVersion = lo self.hi: StandardVersion = hi + self._string: Optional[str] = None + self._hash: Optional[int] = None @classmethod def from_version_range(cls, lo: StandardVersion, hi: StandardVersion) -> "ClosedOpenRange": """Construct ClosedOpenRange from lo:hi range.""" try: - return ClosedOpenRange(lo, _next_version(hi)) + r = ClosedOpenRange(lo, _next_version(hi)) except EmptyRangeError as e: raise EmptyRangeError(f"{lo}:{hi} is an empty range") from e + # Cache hash and string representation + r._hash = hash((lo, hi)) + r._string = _str_range(lo, hi) + return r + def __str__(self) -> str: - # This simplifies 3.1:<3.2 to 3.1:3.1 to 3.1 - # 3:3 -> 3 - hi_prev = _prev_version(self.hi) - if self.lo != StandardVersion.typemin() and self.lo == hi_prev: - return str(self.lo) - lhs = "" if self.lo == StandardVersion.typemin() else str(self.lo) - rhs = "" if hi_prev == StandardVersion.typemax() else str(hi_prev) - return f"{lhs}:{rhs}" + if self._string: + return self._string + self._string = _str_range(self.lo, _prev_version(self.hi)) + return self._string def __repr__(self): return str(self) def __hash__(self): - # prev_version for backward compat. - return hash((self.lo, _prev_version(self.hi))) + if self._hash is not None: + return self._hash + self._hash = hash((self.lo, _prev_version(self.hi))) + return self._hash def __eq__(self, other): - if isinstance(other, StandardVersion): - return False if isinstance(other, ClosedOpenRange): return (self.lo, self.hi) == (other.lo, other.hi) + if isinstance(other, StandardVersion): + return False return NotImplemented def __ne__(self, other): @@ -822,10 +843,10 @@ def __ne__(self, other): return NotImplemented def __lt__(self, other): - if isinstance(other, StandardVersion): - return other > self if isinstance(other, ClosedOpenRange): return (self.lo, self.hi) < (other.lo, other.hi) + if isinstance(other, StandardVersion): + return other > self return NotImplemented def __le__(self, other): @@ -857,10 +878,10 @@ def __contains__(rhs, lhs): def intersects(self, other: VersionType) -> bool: if isinstance(other, StandardVersion): return self.lo <= other < self.hi - if isinstance(other, GitVersion): - return self.lo <= other.ref_version < self.hi if isinstance(other, ClosedOpenRange): return (self.lo < other.hi) and (other.lo < self.hi) + if isinstance(other, GitVersion): + return self.lo <= other.ref_version < self.hi if isinstance(other, VersionList): return any(self.intersects(rhs) for rhs in other) raise TypeError(f"'intersects' not supported for instances of {type(other)}") @@ -877,12 +898,6 @@ def satisfies(self, other: VersionType) -> bool: def _union_if_not_disjoint(self, other: VersionType) -> Optional["ClosedOpenRange"]: """Same as union, but returns None when the union is not connected. This function is not implemented for version lists as right-hand side, as that makes little sense.""" - if isinstance(other, StandardVersion): - return self if self.lo <= other < self.hi else None - - if isinstance(other, GitVersion): - return self if self.lo <= other.ref_version < self.hi else None - if isinstance(other, ClosedOpenRange): # Notice <= cause we want union(1:2, 3:4) = 1:4. return ( @@ -891,6 +906,12 @@ def _union_if_not_disjoint(self, other: VersionType) -> Optional["ClosedOpenRang else None ) + if isinstance(other, StandardVersion): + return self if self.lo <= other < self.hi else None + + if isinstance(other, GitVersion): + return self if self.lo <= other.ref_version < self.hi else None + raise TypeError(f"'union()' not supported for instances of {type(other)}") def union(self, other: VersionType) -> VersionType: @@ -922,22 +943,22 @@ class VersionList(VersionType): versions: List[VersionType] def __init__(self, vlist: Optional[Union[str, VersionType, Iterable]] = None): - if vlist is None: - self.versions = [] - - elif isinstance(vlist, str): + if isinstance(vlist, str): vlist = from_string(vlist) if isinstance(vlist, VersionList): self.versions = vlist.versions else: self.versions = [vlist] - elif isinstance(vlist, (ConcreteVersion, ClosedOpenRange)): - self.versions = [vlist] + elif vlist is None: + self.versions = [] elif isinstance(vlist, VersionList): self.versions = vlist[:] + elif isinstance(vlist, (ConcreteVersion, ClosedOpenRange)): + self.versions = [vlist] + elif isinstance(vlist, Iterable): self.versions = [] for v in vlist: @@ -947,15 +968,7 @@ def __init__(self, vlist: Optional[Union[str, VersionType, Iterable]] = None): raise TypeError(f"Cannot construct VersionList from {type(vlist)}") def add(self, item: VersionType) -> None: - if isinstance(item, (StandardVersion, GitVersion)): - i = bisect_left(self, item) - # Only insert when prev and next are not intersected. - if (i == 0 or not item.intersects(self[i - 1])) and ( - i == len(self) or not item.intersects(self[i]) - ): - self.versions.insert(i, item) - - elif isinstance(item, ClosedOpenRange): + if isinstance(item, ClosedOpenRange): i = bisect_left(self, item) # Note: can span multiple concrete versions to the left (as well as to the right). @@ -982,6 +995,14 @@ def add(self, item: VersionType) -> None: for v in item: self.add(v) + elif isinstance(item, (StandardVersion, GitVersion)): + i = bisect_left(self, item) + # Only insert when prev and next are not intersected. + if (i == 0 or not item.intersects(self[i - 1])) and ( + i == len(self) or not item.intersects(self[i]) + ): + self.versions.insert(i, item) + else: raise TypeError("Can't add %s to VersionList" % type(item)) @@ -1038,6 +1059,9 @@ def satisfies(self, other: VersionType) -> bool: raise TypeError(f"'satisfies()' not supported for instances of {type(other)}") def intersects(self, other: VersionType) -> bool: + if isinstance(other, (ClosedOpenRange, StandardVersion)): + return any(v.intersects(other) for v in self) + if isinstance(other, VersionList): s = o = 0 while s < len(self) and o < len(other): @@ -1049,9 +1073,6 @@ def intersects(self, other: VersionType) -> bool: o += 1 return False - if isinstance(other, (ClosedOpenRange, StandardVersion)): - return any(v.intersects(other) for v in self) - raise TypeError(f"'intersects()' not supported for instances of {type(other)}") def to_dict(self) -> Dict: @@ -1297,13 +1318,13 @@ def from_string(string: str) -> VersionType: # VersionList if "," in string: - return VersionList(list(map(from_string, string.split(",")))) + return VersionList([from_string(x) for x in string.split(",")]) # ClosedOpenRange elif ":" in string: s, e = string.split(":") - lo = StandardVersion.typemin() if s == "" else StandardVersion.from_string(s) - hi = StandardVersion.typemax() if e == "" else StandardVersion.from_string(e) + lo = _STANDARD_VERSION_TYPEMIN if s == "" else StandardVersion.from_string(s) + hi = _STANDARD_VERSION_TYPEMAX if e == "" else StandardVersion.from_string(e) return VersionRange(lo, hi) # StandardVersion diff --git a/pyproject.toml b/pyproject.toml index 8e24a4d3de7c6c..abd0de2db0c311 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,10 +17,8 @@ dev = [ "pytest", "pytest-xdist", "click", - "black", + "ruff", "mypy", - "isort", - "flake8", "vermin", ] ci = ["pytest-cov", "codecov[toml]"] @@ -57,19 +55,27 @@ test = "./bin/spack unit-test" features = ["dev", "ci"] [tool.ruff] +fix = true +target-version = "py37" line-length = 99 extend-include = ["bin/spack"] -extend-exclude = [ +exclude = [ + "var/spack/environments", "lib/spack/spack/vendor", - "*.pyi", + "opt/spack", ] +src = ["lib"] [tool.ruff.format] skip-magic-trailing-comma = true +quote-style = "double" +indent-style = "space" +line-ending = "lf" [tool.ruff.lint] -extend-select = ["I"] -ignore = ["E731", "E203"] +select = ["E", "F", "W", "I"] +ignore = ["E203", "E731", "F811"] +typing-modules = ["spack.vendor.typing_extensions"] [tool.ruff.lint.isort] split-on-trailing-comma = false @@ -77,40 +83,20 @@ section-order = [ "future", "standard-library", "third-party", + "vendor", "spack", "first-party", "local-folder", ] [tool.ruff.lint.isort.sections] +vendor = ["spack.vendor"] spack = ["spack"] [tool.ruff.lint.per-file-ignores] "var/spack/*/package.py" = ["F403", "F405", "F811", "F821"] "*-ci-package.py" = ["F403", "F405", "F821"] -[tool.black] -line-length = 99 -include = "(lib/spack|var/spack/test_repos)/.*\\.pyi?$|bin/spack$" -extend-exclude = "lib/spack/spack/vendor" -skip-magic-trailing-comma = true - -[tool.isort] -line_length = 99 -profile = "black" -sections = [ - "FUTURE", - "STDLIB", - "THIRDPARTY", - "VENDOR", - "FIRSTPARTY", - "LOCALFOLDER", -] -known_first_party = "spack" -known_vendor = ["spack.vendor"] -skip = ["lib/spack/spack/vendor"] -src_paths = "lib" -honor_noqa = true [tool.mypy] files = ["lib/spack/spack/**/*.py"] diff --git a/share/spack/bash/spack-completion.bash b/share/spack/bash/spack-completion.bash index e524565f41de0e..cf4b273937177d 100755 --- a/share/spack/bash/spack-completion.bash +++ b/share/spack/bash/spack-completion.bash @@ -70,28 +70,30 @@ _bash_completion_spack() { # In all following examples, let the cursor be denoted by brackets, i.e. [] # For our purposes, flags should not affect tab completion. For instance, - # `spack install []` and `spack -d install --jobs 8 []` should both give the same - # possible completions. Therefore, we need to ignore any flags in COMP_WORDS. + # `spack install []` and `spack -d install --jobs 8 []` should both give the + # same possible completions. Therefore, we need to ignore any flags in + # COMP_WORDS. We do this by navigating the subcommand tree level-by-level: a + # non-flag word is only kept if a completion function exists for the + # resulting path. local -a COMP_WORDS_NO_FLAGS - local index=0 + COMP_WORDS_NO_FLAGS=("spack") + local subfunction="_spack" + local index=1 while [[ "$index" -lt "$COMP_CWORD" ]] do - if [[ "${COMP_WORDS[$index]}" == [a-z]* ]] + local word="${COMP_WORDS[$index]}" + if [[ "$word" != -* ]] then - COMP_WORDS_NO_FLAGS+=("${COMP_WORDS[$index]}") + local candidate="${subfunction}_${word//-/_}" + if declare -f "$candidate" > /dev/null 2>&1 + then + COMP_WORDS_NO_FLAGS+=("$word") + subfunction="$candidate" + fi fi - let index++ + ((index++)) done - # Options will be listed by a subfunction named after non-flag arguments. - # For example, `spack -d install []` will call _spack_install - # and `spack compiler add []` will call _spack_compiler_add - local subfunction=$(IFS='_'; echo "_${COMP_WORDS_NO_FLAGS[*]}") - - # Translate dashes to underscores, as dashes are not permitted in - # compatibility mode. See https://github.com/spack/spack/pull/4079 - subfunction=${subfunction//-/_} - # However, the word containing the current cursor position needs to be # added regardless of whether or not it is a flag. This allows us to # complete something like `spack install --keep-st[]` @@ -102,7 +104,7 @@ _bash_completion_spack() { local COMP_CWORD_NO_FLAGS=$((${#COMP_WORDS_NO_FLAGS[@]} - 1)) # There is no guarantee that the cursor is at the end of the command line - # when tab completion is envoked. For example, in the following situation: + # when tab completion is invoked. For example, in the following situation: # `spack -d [] install` # if the user presses the TAB key, a list of valid flags should be listed. # Note that we cannot simply ignore everything after the cursor. In the @@ -117,7 +119,7 @@ _bash_completion_spack() { list_options=true fi - # In general, when envoking tab completion, the user is not expecting to + # In general, when invoking tab completion, the user is not expecting to # see optional flags mixed in with subcommands or package names. Tab # completion is used by those who are either lazy or just bad at spelling. # If someone doesn't remember what flag to use, seeing single letter flags @@ -144,14 +146,8 @@ _bash_completion_spack() { # Uncomment this line to enable logging #_test_vars >> temp - # Make sure function exists before calling it - local rgx #this dance is necessary to cover bash and zsh regex - rgx="$subfunction.*function.* " - if [[ "$(LC_ALL=C type $subfunction 2>&1)" =~ $rgx ]] - then - $subfunction - COMPREPLY=($(_compgen_w "$SPACK_COMPREPLY" "$cur")) - fi + $subfunction + COMPREPLY=($(_compgen_w "$SPACK_COMPREPLY" "$cur")) # if every completion is an alias for the same thing, just return that thing. _spack_compress_aliases @@ -195,7 +191,7 @@ _installed_packages() { _installed_compilers() { if [[ -z "${SPACK_INSTALLED_COMPILERS:-}" ]] then - SPACK_INSTALLED_COMPILERS="$(spack compilers | egrep -v "^(-|=)")" + SPACK_INSTALLED_COMPILERS="$(spack compilers)" fi SPACK_COMPREPLY="$SPACK_INSTALLED_COMPILERS" } diff --git a/share/spack/fish/spack-completion.fish b/share/spack/fish/spack-completion.fish index b389e1c114ba0a..85e86c0a79256e 100644 --- a/share/spack/fish/spack-completion.fish +++ b/share/spack/fish/spack-completion.fish @@ -193,7 +193,7 @@ function __fish_spack_gpg_keys end function __fish_spack_installed_compilers - spack compilers | grep -v '^[=-]\|^$' + spack compilers end function __fish_spack_installed_packages diff --git a/share/spack/qa/config_state.py b/share/spack/qa/config_state.py index 027c5672002655..c2572138c62762 100644 --- a/share/spack/qa/config_state.py +++ b/share/spack/qa/config_state.py @@ -6,6 +6,7 @@ The option `config:cache` is supposed to be False, and overridden to True from the command line. """ + import multiprocessing as mp import spack.config diff --git a/share/spack/qa/run-unit-tests b/share/spack/qa/run-unit-tests index cc86de6f4ef16e..d444bc71188940 100755 --- a/share/spack/qa/run-unit-tests +++ b/share/spack/qa/run-unit-tests @@ -35,8 +35,6 @@ spack config get compilers bin/spack -h bin/spack help -a -# Profile and print top 20 lines for a simple call to spack spec -spack -p --lines 20 spec mpileaks%gcc $coverage_run $(which spack) bootstrap status --dev --optional # Check that we can import Spack packages directly as a first import diff --git a/share/spack/setup-env.bat b/share/spack/setup-env.bat index c3b91ece1fccdf..0c282427b6f192 100644 --- a/share/spack/setup-env.bat +++ b/share/spack/setup-env.bat @@ -56,10 +56,6 @@ if defined py_path ( set "PATH=%py_path%;%PATH%" ) -if defined py_exe ( - "%py_exe%" "%SPACK_ROOT%\bin\haspywin.py" -) - if not defined EDITOR ( set EDITOR=notepad ) diff --git a/share/spack/setup-env.fish b/share/spack/setup-env.fish index d2447a26915a49..6319dadda2840a 100644 --- a/share/spack/setup-env.fish +++ b/share/spack/setup-env.fish @@ -716,19 +716,9 @@ set -xg _sp_shell "fish" -if test -z "$SPACK_SKIP_MODULES" +if test -z "$SPACK_SKIP_MODULES"; and begin; type -q module; or type -q use; end # - # Check whether we need environment-variables (module) <= `use` is not available - # - set -l need_module "no" - if not functions -q use; and not functions -q module - set need_module "yes" - end - - - - # - # Make environment-modules available to shell + # Make shell vars available to fish # function sp_apply_shell_vars -d "applies expressions of the type `a='b'` as `set a b`" @@ -740,34 +730,10 @@ if test -z "$SPACK_SKIP_MODULES" set -xg $expr_token[1] (string split ":" $expr_token[2]) end + set -l sp_shell_vars (command spack --print-shell-vars sh) - if test "$need_module" = "yes" - set -l sp_shell_vars (command spack --print-shell-vars sh,modules) - - for sp_var_expr in $sp_shell_vars - sp_apply_shell_vars $sp_var_expr - end - - # _sp_module_prefix is set by spack --print-shell-vars - if test "$_sp_module_prefix" != "not_installed" - set -xg MODULE_PREFIX $_sp_module_prefix - spack_pathadd PATH "$MODULE_PREFIX/bin" - end - - else - - set -l sp_shell_vars (command spack --print-shell-vars sh) - - for sp_var_expr in $sp_shell_vars - sp_apply_shell_vars $sp_var_expr - end - - end - - if test "$need_module" = "yes" - function module -d "wrapper for the `module` command to point at Spack's modules instance" --inherit-variable MODULE_PREFIX - eval $MODULE_PREFIX/bin/modulecmd $SPACK_SHELL $argv - end + for sp_var_expr in $sp_shell_vars + sp_apply_shell_vars $sp_var_expr end diff --git a/share/spack/setup-env.ps1 b/share/spack/setup-env.ps1 index 99683099f8a4d9..de609e1e4d9d34 100644 --- a/share/spack/setup-env.ps1 +++ b/share/spack/setup-env.ps1 @@ -11,7 +11,7 @@ Pop-Location Set-Variable -Name python_pf_ver -Value (Get-Command -Name python -ErrorAction SilentlyContinue).Path # If python_pf_ver is not defined, we cannot find Python on the Path -# We next look for Spack vendored copys +# We next look for Spack vendored copies if ($null -eq $python_pf_ver) { $python_pf_ver_list = Resolve-Path -Path "$PWD\Python*" @@ -34,11 +34,6 @@ if (!$null -eq $py_path) $Env:Path = "$py_path;$Env:Path" } -if (!$null -eq $py_exe) -{ - & "$py_exe" "$Env:SPACK_ROOT\bin\haspywin.py" -} - $Env:Path = "$Env:SPACK_ROOT\bin;$Env:Path" if ($null -eq $Env:EDITOR) { diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index dd71d33420e009..e7ab2e853459a2 100644 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -232,6 +232,10 @@ _spack_determine_shell() { # If procfs is present this seems a more reliable # way to detect the current shell _sp_exe=$(readlink /proc/$$/exe) + # Qemu emulation has _sp_exe point to the emulator + if [ "${_sp_exe##*qemu*}" != "${_sp_exe}" ]; then + _sp_exe=$(cat /proc/$$/comm) + fi # Shell may contain number, like zsh5 instead of zsh basename ${_sp_exe} | tr -d '0123456789' elif [ -n "${BASH:-}" ]; then @@ -305,13 +309,6 @@ else fi _spack_pathadd PATH "${_sp_prefix%/}/bin" -# -# Check whether a function of the given name is defined -# -_spack_fn_exists() { - LANG= type $1 2>&1 | grep -q 'function' -} - # Define the spack shell function with some informative no-ops, so when users # run `which spack`, they see the path to spack and where the function is from. eval "spack() { @@ -335,39 +332,9 @@ for cmd in "${SPACK_PYTHON:-}" python3 python python2; do fi done -if [ -z "${SPACK_SKIP_MODULES+x}" ]; then - need_module="no" - if ! _spack_fn_exists use && ! _spack_fn_exists module; then - need_module="yes" - fi; - - # - # make available environment-modules - # - if [ "${need_module}" = "yes" ]; then - eval `spack --print-shell-vars sh,modules` - - # _sp_module_prefix is set by spack --print-sh-vars - if [ "${_sp_module_prefix}" != "not_installed" ]; then - # activate it! - # environment-modules@4: has a bin directory inside its prefix - _sp_module_bin="${_sp_module_prefix}/bin" - if [ ! -d "${_sp_module_bin}" ]; then - # environment-modules@3 has a nested bin directory - _sp_module_bin="${_sp_module_prefix}/Modules/bin" - fi - - # _sp_module_bin and _sp_shell are evaluated here; the quoted - # eval statement and $* are deferred. - _sp_cmd="module() { eval \`${_sp_module_bin}/modulecmd ${_sp_shell} \$*\`; }" - eval "$_sp_cmd" - _spack_pathadd PATH "${_sp_module_bin}" - fi; - else - stdout="$(command spack --print-shell-vars sh)" || return - eval "$stdout" - fi; - +if [ -z "${SPACK_SKIP_MODULES+x}" ] && { type module > /dev/null 2>&1 || type use > /dev/null 2>&1; }; then + stdout="$(command spack --print-shell-vars sh)" || return + eval "$stdout" # # set module system roots diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index ecfb9b01c23244..315bc0edf75237 100644 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -70,28 +70,30 @@ _bash_completion_spack() { # In all following examples, let the cursor be denoted by brackets, i.e. [] # For our purposes, flags should not affect tab completion. For instance, - # `spack install []` and `spack -d install --jobs 8 []` should both give the same - # possible completions. Therefore, we need to ignore any flags in COMP_WORDS. + # `spack install []` and `spack -d install --jobs 8 []` should both give the + # same possible completions. Therefore, we need to ignore any flags in + # COMP_WORDS. We do this by navigating the subcommand tree level-by-level: a + # non-flag word is only kept if a completion function exists for the + # resulting path. local -a COMP_WORDS_NO_FLAGS - local index=0 + COMP_WORDS_NO_FLAGS=("spack") + local subfunction="_spack" + local index=1 while [[ "$index" -lt "$COMP_CWORD" ]] do - if [[ "${COMP_WORDS[$index]}" == [a-z]* ]] + local word="${COMP_WORDS[$index]}" + if [[ "$word" != -* ]] then - COMP_WORDS_NO_FLAGS+=("${COMP_WORDS[$index]}") + local candidate="${subfunction}_${word//-/_}" + if declare -f "$candidate" > /dev/null 2>&1 + then + COMP_WORDS_NO_FLAGS+=("$word") + subfunction="$candidate" + fi fi - let index++ + ((index++)) done - # Options will be listed by a subfunction named after non-flag arguments. - # For example, `spack -d install []` will call _spack_install - # and `spack compiler add []` will call _spack_compiler_add - local subfunction=$(IFS='_'; echo "_${COMP_WORDS_NO_FLAGS[*]}") - - # Translate dashes to underscores, as dashes are not permitted in - # compatibility mode. See https://github.com/spack/spack/pull/4079 - subfunction=${subfunction//-/_} - # However, the word containing the current cursor position needs to be # added regardless of whether or not it is a flag. This allows us to # complete something like `spack install --keep-st[]` @@ -102,7 +104,7 @@ _bash_completion_spack() { local COMP_CWORD_NO_FLAGS=$((${#COMP_WORDS_NO_FLAGS[@]} - 1)) # There is no guarantee that the cursor is at the end of the command line - # when tab completion is envoked. For example, in the following situation: + # when tab completion is invoked. For example, in the following situation: # `spack -d [] install` # if the user presses the TAB key, a list of valid flags should be listed. # Note that we cannot simply ignore everything after the cursor. In the @@ -117,7 +119,7 @@ _bash_completion_spack() { list_options=true fi - # In general, when envoking tab completion, the user is not expecting to + # In general, when invoking tab completion, the user is not expecting to # see optional flags mixed in with subcommands or package names. Tab # completion is used by those who are either lazy or just bad at spelling. # If someone doesn't remember what flag to use, seeing single letter flags @@ -144,14 +146,8 @@ _bash_completion_spack() { # Uncomment this line to enable logging #_test_vars >> temp - # Make sure function exists before calling it - local rgx #this dance is necessary to cover bash and zsh regex - rgx="$subfunction.*function.* " - if [[ "$(LC_ALL=C type $subfunction 2>&1)" =~ $rgx ]] - then - $subfunction - COMPREPLY=($(_compgen_w "$SPACK_COMPREPLY" "$cur")) - fi + $subfunction + COMPREPLY=($(_compgen_w "$SPACK_COMPREPLY" "$cur")) # if every completion is an alias for the same thing, just return that thing. _spack_compress_aliases @@ -195,7 +191,7 @@ _installed_packages() { _installed_compilers() { if [[ -z "${SPACK_INSTALLED_COMPILERS:-}" ]] then - SPACK_INSTALLED_COMPILERS="$(spack compilers | egrep -v "^(-|=)")" + SPACK_INSTALLED_COMPILERS="$(spack compilers)" fi SPACK_COMPREPLY="$SPACK_INSTALLED_COMPILERS" } @@ -570,7 +566,7 @@ _spack_buildcache() { _spack_buildcache_push() { if $list_options then - SPACK_COMPREPLY="-h --help -f --force --unsigned -u --signed --key -k --update-index --rebuild-index --only --with-build-dependencies --without-build-dependencies --fail-fast --base-image --tag -t --private -j --jobs" + SPACK_COMPREPLY="-h --help -f --force --unsigned -u --signed --key -k --update-index --rebuild-index --only --with-build-dependencies --without-build-dependencies --fail-fast --allow-missing --base-image --tag -t --private --group -j --jobs" else _mirrors fi @@ -579,7 +575,7 @@ _spack_buildcache_push() { _spack_buildcache_create() { if $list_options then - SPACK_COMPREPLY="-h --help -f --force --unsigned -u --signed --key -k --update-index --rebuild-index --only --with-build-dependencies --without-build-dependencies --fail-fast --base-image --tag -t --private -j --jobs" + SPACK_COMPREPLY="-h --help -f --force --unsigned -u --signed --key -k --update-index --rebuild-index --only --with-build-dependencies --without-build-dependencies --fail-fast --allow-missing --base-image --tag -t --private --group -j --jobs" else _mirrors fi @@ -681,7 +677,7 @@ _spack_buildcache_migrate() { _spack_cd() { if $list_options then - SPACK_COMPREPLY="-h --help -m --module-dir -r --spack-root -i --install-dir -p --package-dir --repo --packages -P -s --stage-dir -S --stages -c --source-dir -b --build-dir -e --env --first" + SPACK_COMPREPLY="-h --help -m --module-dir -r --spack-root -i --install-dir -p --package-dir --repo --packages -P -s --stage-dir -S --stages -c --source-dir -b --build-dir -e --env -v --view --first" else _all_packages fi @@ -848,7 +844,7 @@ _spack_config() { _spack_config_get() { if $list_options then - SPACK_COMPREPLY="-h --help --json" + SPACK_COMPREPLY="-h --help --json --group" else _config_sections fi @@ -857,7 +853,7 @@ _spack_config_get() { _spack_config_blame() { if $list_options then - SPACK_COMPREPLY="-h --help" + SPACK_COMPREPLY="-h --help --group" else _config_sections fi @@ -1141,7 +1137,7 @@ _spack_env_view() { then SPACK_COMPREPLY="-h --help" else - SPACK_COMPREPLY="" + SPACK_COMPREPLY="disable enable regenerate" fi } @@ -1410,7 +1406,7 @@ _spack_load() { _spack_location() { if $list_options then - SPACK_COMPREPLY="-h --help -m --module-dir -r --spack-root -i --install-dir -p --package-dir --repo --packages -P -s --stage-dir -S --stages -c --source-dir -b --build-dir -e --env --first" + SPACK_COMPREPLY="-h --help -m --module-dir -r --spack-root -i --install-dir -p --package-dir --repo --packages -P -s --stage-dir -S --stages -c --source-dir -b --build-dir -e --env -v --view --first" else _all_packages fi @@ -1419,7 +1415,7 @@ _spack_location() { _spack_log_parse() { if $list_options then - SPACK_COMPREPLY="-h --help --show -c --context -p --profile -w --width -j --jobs" + SPACK_COMPREPLY="-h --help --show -c --context -p --profile -w --width -j --jobs -t --tail" else SPACK_COMPREPLY="" fi @@ -1806,7 +1802,7 @@ _spack_repo() { then SPACK_COMPREPLY="-h --help" else - SPACK_COMPREPLY="create list ls add set remove rm migrate update" + SPACK_COMPREPLY="create list ls add set remove rm migrate update show-version-updates" fi } @@ -1820,11 +1816,11 @@ _spack_repo_create() { } _spack_repo_list() { - SPACK_COMPREPLY="-h --help --scope --names --namespaces" + SPACK_COMPREPLY="-h --help --scope --names --namespaces --json" } _spack_repo_ls() { - SPACK_COMPREPLY="-h --help --scope --names --namespaces" + SPACK_COMPREPLY="-h --help --scope --names --namespaces --json" } _spack_repo_add() { @@ -1881,6 +1877,15 @@ _spack_repo_update() { fi } +_spack_repo_show_version_updates() { + if $list_options + then + SPACK_COMPREPLY="-h --help --no-manual-packages --no-git-versions --only-redistributable" + else + SPACK_COMPREPLY="" + fi +} + _spack_resource() { if $list_options then diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index c740edfa425e10..5a0a496923cdf7 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -193,7 +193,7 @@ function __fish_spack_gpg_keys end function __fish_spack_installed_compilers - spack compilers | grep -v '^[=-]\|^$' + spack compilers end function __fish_spack_installed_packages @@ -458,7 +458,6 @@ complete -c spack -n '__fish_spack_using_command ' -s d -l debug -d 'write out d complete -c spack -n '__fish_spack_using_command ' -s t -l backtrace -f -a backtrace complete -c spack -n '__fish_spack_using_command ' -s t -l backtrace -d 'always show backtraces for exceptions' complete -c spack -n '__fish_spack_using_command ' -l pdb -f -a pdb -complete -c spack -n '__fish_spack_using_command ' -l pdb -d 'run spack under the pdb debugger' complete -c spack -n '__fish_spack_using_command ' -l timestamp -f -a timestamp complete -c spack -n '__fish_spack_using_command ' -l timestamp -d 'add a timestamp to tty output' complete -c spack -n '__fish_spack_using_command ' -s m -l mock -f -a mock @@ -472,13 +471,9 @@ complete -c spack -n '__fish_spack_using_command ' -s l -l enable-locks -d 'use complete -c spack -n '__fish_spack_using_command ' -s L -l disable-locks -f -a locks complete -c spack -n '__fish_spack_using_command ' -s L -l disable-locks -d 'do not use filesystem locking (unsafe)' complete -c spack -n '__fish_spack_using_command ' -s p -l profile -f -a spack_profile -complete -c spack -n '__fish_spack_using_command ' -s p -l profile -d 'profile execution using cProfile' complete -c spack -n '__fish_spack_using_command ' -l profile-file -r -f -a profile_file -complete -c spack -n '__fish_spack_using_command ' -l profile-file -r -d 'Filename to save profile data to.' complete -c spack -n '__fish_spack_using_command ' -l sorted-profile -r -f -a sorted_profile -complete -c spack -n '__fish_spack_using_command ' -l sorted-profile -r -d 'profile and sort by STAT, which can be: calls, ncalls,' complete -c spack -n '__fish_spack_using_command ' -l lines -r -f -a lines -complete -c spack -n '__fish_spack_using_command ' -l lines -r -d 'lines of profile output or '"'"'all'"'"' (default: 20)' # spack add set -g __fish_spack_optspecs_spack_add h/help l/list-name= @@ -707,7 +702,7 @@ complete -c spack -n '__fish_spack_using_command buildcache' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command buildcache' -s h -l help -d 'show this help message and exit' # spack buildcache push -set -g __fish_spack_optspecs_spack_buildcache_push h/help f/force u/unsigned signed k/key= update-index only= with-build-dependencies without-build-dependencies fail-fast base-image= t/tag= private j/jobs= +set -g __fish_spack_optspecs_spack_buildcache_push h/help f/force u/unsigned signed k/key= update-index only= with-build-dependencies without-build-dependencies fail-fast allow-missing base-image= t/tag= private group= j/jobs= complete -c spack -n '__fish_spack_using_command_pos_remainder 1 buildcache push' -f -k -a '(__fish_spack_specs)' complete -c spack -n '__fish_spack_using_command buildcache push' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command buildcache push' -s h -l help -d 'show this help message and exit' @@ -729,17 +724,21 @@ complete -c spack -n '__fish_spack_using_command buildcache push' -l without-bui complete -c spack -n '__fish_spack_using_command buildcache push' -l without-build-dependencies -d 'exclude build dependencies from the buildcache' complete -c spack -n '__fish_spack_using_command buildcache push' -l fail-fast -f -a fail_fast complete -c spack -n '__fish_spack_using_command buildcache push' -l fail-fast -d 'stop pushing on first failure (default is best effort)' +complete -c spack -n '__fish_spack_using_command buildcache push' -l allow-missing -f -a allow_missing +complete -c spack -n '__fish_spack_using_command buildcache push' -l allow-missing -d 'allow not installed specs to continue without failure (default fails on missing specs)' complete -c spack -n '__fish_spack_using_command buildcache push' -l base-image -r -f -a base_image complete -c spack -n '__fish_spack_using_command buildcache push' -l base-image -r -d 'specify the base image for the buildcache' complete -c spack -n '__fish_spack_using_command buildcache push' -l tag -s t -r -f -a tag complete -c spack -n '__fish_spack_using_command buildcache push' -l tag -s t -r -d 'when pushing to an OCI registry, tag an image containing all root specs and their runtime dependencies' complete -c spack -n '__fish_spack_using_command buildcache push' -l private -f -a private complete -c spack -n '__fish_spack_using_command buildcache push' -l private -d 'for a private mirror, include non-redistributable packages' +complete -c spack -n '__fish_spack_using_command buildcache push' -l group -r -f -a groups +complete -c spack -n '__fish_spack_using_command buildcache push' -l group -r -d 'push only specs from the given environment group (can be specified multiple times, requires an active environment)' complete -c spack -n '__fish_spack_using_command buildcache push' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command buildcache push' -s j -l jobs -r -d 'explicitly set number of parallel jobs' # spack buildcache create -set -g __fish_spack_optspecs_spack_buildcache_create h/help f/force u/unsigned signed k/key= update-index only= with-build-dependencies without-build-dependencies fail-fast base-image= t/tag= private j/jobs= +set -g __fish_spack_optspecs_spack_buildcache_create h/help f/force u/unsigned signed k/key= update-index only= with-build-dependencies without-build-dependencies fail-fast allow-missing base-image= t/tag= private group= j/jobs= complete -c spack -n '__fish_spack_using_command_pos_remainder 1 buildcache create' -f -k -a '(__fish_spack_specs)' complete -c spack -n '__fish_spack_using_command buildcache create' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command buildcache create' -s h -l help -d 'show this help message and exit' @@ -761,12 +760,16 @@ complete -c spack -n '__fish_spack_using_command buildcache create' -l without-b complete -c spack -n '__fish_spack_using_command buildcache create' -l without-build-dependencies -d 'exclude build dependencies from the buildcache' complete -c spack -n '__fish_spack_using_command buildcache create' -l fail-fast -f -a fail_fast complete -c spack -n '__fish_spack_using_command buildcache create' -l fail-fast -d 'stop pushing on first failure (default is best effort)' +complete -c spack -n '__fish_spack_using_command buildcache create' -l allow-missing -f -a allow_missing +complete -c spack -n '__fish_spack_using_command buildcache create' -l allow-missing -d 'allow not installed specs to continue without failure (default fails on missing specs)' complete -c spack -n '__fish_spack_using_command buildcache create' -l base-image -r -f -a base_image complete -c spack -n '__fish_spack_using_command buildcache create' -l base-image -r -d 'specify the base image for the buildcache' complete -c spack -n '__fish_spack_using_command buildcache create' -l tag -s t -r -f -a tag complete -c spack -n '__fish_spack_using_command buildcache create' -l tag -s t -r -d 'when pushing to an OCI registry, tag an image containing all root specs and their runtime dependencies' complete -c spack -n '__fish_spack_using_command buildcache create' -l private -f -a private complete -c spack -n '__fish_spack_using_command buildcache create' -l private -d 'for a private mirror, include non-redistributable packages' +complete -c spack -n '__fish_spack_using_command buildcache create' -l group -r -f -a groups +complete -c spack -n '__fish_spack_using_command buildcache create' -l group -r -d 'push only specs from the given environment group (can be specified multiple times, requires an active environment)' complete -c spack -n '__fish_spack_using_command buildcache create' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command buildcache create' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -881,7 +884,7 @@ complete -c spack -n '__fish_spack_using_command buildcache update-index' -s h - complete -c spack -n '__fish_spack_using_command buildcache update-index' -l name -s n -r -f -a name complete -c spack -n '__fish_spack_using_command buildcache update-index' -l name -s n -r -d 'Name of the view index to update' complete -c spack -n '__fish_spack_using_command buildcache update-index' -l append -s a -f -a append -complete -c spack -n '__fish_spack_using_command buildcache update-index' -l append -s a -d 'Append the listed specs to the current view index if it already exists. This operation does not guarentee atomic write and should be run with care.' +complete -c spack -n '__fish_spack_using_command buildcache update-index' -l append -s a -d 'Append the listed specs to the current view index if it already exists. This operation does not guarantee atomic write and should be run with care.' complete -c spack -n '__fish_spack_using_command buildcache update-index' -l force -s f -f -a force complete -c spack -n '__fish_spack_using_command buildcache update-index' -l force -s f -d 'If a view index already exists, overwrite it and suppress warnings (this is the default for non-view indices)' complete -c spack -n '__fish_spack_using_command buildcache update-index' -s k -l keys -f -a keys @@ -897,7 +900,7 @@ complete -c spack -n '__fish_spack_using_command buildcache rebuild-index' -s h complete -c spack -n '__fish_spack_using_command buildcache rebuild-index' -l name -s n -r -f -a name complete -c spack -n '__fish_spack_using_command buildcache rebuild-index' -l name -s n -r -d 'Name of the view index to update' complete -c spack -n '__fish_spack_using_command buildcache rebuild-index' -l append -s a -f -a append -complete -c spack -n '__fish_spack_using_command buildcache rebuild-index' -l append -s a -d 'Append the listed specs to the current view index if it already exists. This operation does not guarentee atomic write and should be run with care.' +complete -c spack -n '__fish_spack_using_command buildcache rebuild-index' -l append -s a -d 'Append the listed specs to the current view index if it already exists. This operation does not guarantee atomic write and should be run with care.' complete -c spack -n '__fish_spack_using_command buildcache rebuild-index' -l force -s f -f -a force complete -c spack -n '__fish_spack_using_command buildcache rebuild-index' -l force -s f -d 'If a view index already exists, overwrite it and suppress warnings (this is the default for non-view indices)' complete -c spack -n '__fish_spack_using_command buildcache rebuild-index' -s k -l keys -f -a keys @@ -918,7 +921,7 @@ complete -c spack -n '__fish_spack_using_command buildcache migrate' -s y -l yes complete -c spack -n '__fish_spack_using_command buildcache migrate' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request' # spack cd -set -g __fish_spack_optspecs_spack_cd h/help m/module-dir r/spack-root i/install-dir p/package-dir repo= s/stage-dir S/stages c/source-dir b/build-dir e/env= first +set -g __fish_spack_optspecs_spack_cd h/help m/module-dir r/spack-root i/install-dir p/package-dir repo= s/stage-dir S/stages c/source-dir b/build-dir e/env= v/view= first complete -c spack -n '__fish_spack_using_command_pos_remainder 0 cd' -f -k -a '(__fish_spack_specs)' complete -c spack -n '__fish_spack_using_command cd' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command cd' -s h -l help -d 'show this help message and exit' @@ -942,6 +945,8 @@ complete -c spack -n '__fish_spack_using_command cd' -s b -l build-dir -f -a bui complete -c spack -n '__fish_spack_using_command cd' -s b -l build-dir -d 'build directory for a spec (requires it to be staged first)' complete -c spack -n '__fish_spack_using_command cd' -s e -l env -r -f -a location_env complete -c spack -n '__fish_spack_using_command cd' -s e -l env -r -d 'location of the named or current environment' +complete -c spack -n '__fish_spack_using_command cd' -s v -l view -r -f -a location_view +complete -c spack -n '__fish_spack_using_command cd' -s v -l view -r -d 'location of the named or active environment view' complete -c spack -n '__fish_spack_using_command cd' -l first -f -a find_first complete -c spack -n '__fish_spack_using_command cd' -l first -d 'use the first match if multiple packages match the spec' @@ -1264,18 +1269,22 @@ complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_bui complete -c spack -n '__fish_spack_using_command config' -l scope -r -d 'configuration scope to read/modify' # spack config get -set -g __fish_spack_optspecs_spack_config_get h/help json +set -g __fish_spack_optspecs_spack_config_get h/help json group= complete -c spack -n '__fish_spack_using_command_pos 0 config get' -f -a 'bootstrap cdash ci compilers concretizer config definitions develop env_vars include mirrors modules packages repos toolchains upstreams view' complete -c spack -n '__fish_spack_using_command config get' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config get' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command config get' -l json -f -a json complete -c spack -n '__fish_spack_using_command config get' -l json -d 'output configuration as JSON' +complete -c spack -n '__fish_spack_using_command config get' -l group -r -f -a group +complete -c spack -n '__fish_spack_using_command config get' -l group -r -d 'show configuration as seen by this environment spec group (requires active env)' # spack config blame -set -g __fish_spack_optspecs_spack_config_blame h/help +set -g __fish_spack_optspecs_spack_config_blame h/help group= complete -c spack -n '__fish_spack_using_command_pos 0 config blame' -f -a 'bootstrap cdash ci compilers concretizer config definitions develop env_vars include mirrors modules packages repos toolchains upstreams view' complete -c spack -n '__fish_spack_using_command config blame' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config blame' -s h -l help -d 'show this help message and exit' +complete -c spack -n '__fish_spack_using_command config blame' -l group -r -f -a group +complete -c spack -n '__fish_spack_using_command config blame' -l group -r -d 'show configuration as seen by this environment spec group (requires active env)' # spack config edit set -g __fish_spack_optspecs_spack_config_edit h/help print-file @@ -2253,7 +2262,7 @@ complete -c spack -n '__fish_spack_using_command load' -l list -f -a list complete -c spack -n '__fish_spack_using_command load' -l list -d 'show loaded packages: same as ``spack find --loaded``' # spack location -set -g __fish_spack_optspecs_spack_location h/help m/module-dir r/spack-root i/install-dir p/package-dir repo= s/stage-dir S/stages c/source-dir b/build-dir e/env= first +set -g __fish_spack_optspecs_spack_location h/help m/module-dir r/spack-root i/install-dir p/package-dir repo= s/stage-dir S/stages c/source-dir b/build-dir e/env= v/view= first complete -c spack -n '__fish_spack_using_command_pos_remainder 0 location' -f -k -a '(__fish_spack_specs)' complete -c spack -n '__fish_spack_using_command location' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command location' -s h -l help -d 'show this help message and exit' @@ -2277,11 +2286,13 @@ complete -c spack -n '__fish_spack_using_command location' -s b -l build-dir -f complete -c spack -n '__fish_spack_using_command location' -s b -l build-dir -d 'build directory for a spec (requires it to be staged first)' complete -c spack -n '__fish_spack_using_command location' -s e -l env -r -f -a location_env complete -c spack -n '__fish_spack_using_command location' -s e -l env -r -d 'location of the named or current environment' +complete -c spack -n '__fish_spack_using_command location' -s v -l view -r -f -a location_view +complete -c spack -n '__fish_spack_using_command location' -s v -l view -r -d 'location of the named or active environment view' complete -c spack -n '__fish_spack_using_command location' -l first -f -a find_first complete -c spack -n '__fish_spack_using_command location' -l first -d 'use the first match if multiple packages match the spec' # spack log-parse -set -g __fish_spack_optspecs_spack_log_parse h/help show= c/context= p/profile w/width= j/jobs= +set -g __fish_spack_optspecs_spack_log_parse h/help show= c/context= p/profile w/width= j/jobs= t/tail= complete -c spack -n '__fish_spack_using_command log-parse' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command log-parse' -s h -l help -d 'show this help message and exit' @@ -2292,9 +2303,9 @@ complete -c spack -n '__fish_spack_using_command log-parse' -s c -l context -r - complete -c spack -n '__fish_spack_using_command log-parse' -s p -l profile -f -a profile complete -c spack -n '__fish_spack_using_command log-parse' -s p -l profile -d 'print out a profile of time spent in regexes during parse' complete -c spack -n '__fish_spack_using_command log-parse' -s w -l width -r -f -a width -complete -c spack -n '__fish_spack_using_command log-parse' -s w -l width -r -d 'wrap width: auto-size to terminal by default; 0 for no wrap' complete -c spack -n '__fish_spack_using_command log-parse' -s j -l jobs -r -f -a jobs -complete -c spack -n '__fish_spack_using_command log-parse' -s j -l jobs -r -d 'number of jobs to parse log file (default: 1 for short logs, ncpus for long logs)' +complete -c spack -n '__fish_spack_using_command log-parse' -s t -l tail -r -f -a tail +complete -c spack -n '__fish_spack_using_command log-parse' -s t -l tail -r -d 'number of trailing log lines to show (0 to disable)' # spack logs set -g __fish_spack_optspecs_spack_logs h/help @@ -2838,6 +2849,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 repo' -f -a remove -d 're complete -c spack -n '__fish_spack_using_command_pos 0 repo' -f -a rm -d 'remove a repository from Spack'"'"'s configuration' complete -c spack -n '__fish_spack_using_command_pos 0 repo' -f -a migrate -d 'migrate a package repository to the latest Package API' complete -c spack -n '__fish_spack_using_command_pos 0 repo' -f -a update -d 'update one or more package repositories' +complete -c spack -n '__fish_spack_using_command_pos 0 repo' -f -a show-version-updates -d 'show version specs that were added between two commits' complete -c spack -n '__fish_spack_using_command repo' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo' -s h -l help -d 'show this help message and exit' @@ -2850,7 +2862,7 @@ complete -c spack -n '__fish_spack_using_command repo create' -s d -l subdirecto complete -c spack -n '__fish_spack_using_command repo create' -s d -l subdirectory -r -d 'subdirectory to store packages in the repository' # spack repo list -set -g __fish_spack_optspecs_spack_repo_list h/help scope= names namespaces +set -g __fish_spack_optspecs_spack_repo_list h/help scope= names namespaces json complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' @@ -2859,9 +2871,11 @@ complete -c spack -n '__fish_spack_using_command repo list' -l names -f -a names complete -c spack -n '__fish_spack_using_command repo list' -l names -d 'show configuration names only' complete -c spack -n '__fish_spack_using_command repo list' -l namespaces -f -a namespaces complete -c spack -n '__fish_spack_using_command repo list' -l namespaces -d 'show repository namespaces only' +complete -c spack -n '__fish_spack_using_command repo list' -l json -f -a json +complete -c spack -n '__fish_spack_using_command repo list' -l json -d 'output repositories as machine-readable json records' # spack repo ls -set -g __fish_spack_optspecs_spack_repo_ls h/help scope= names namespaces +set -g __fish_spack_optspecs_spack_repo_ls h/help scope= names namespaces json complete -c spack -n '__fish_spack_using_command repo ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo ls' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' @@ -2870,6 +2884,8 @@ complete -c spack -n '__fish_spack_using_command repo ls' -l names -f -a names complete -c spack -n '__fish_spack_using_command repo ls' -l names -d 'show configuration names only' complete -c spack -n '__fish_spack_using_command repo ls' -l namespaces -f -a namespaces complete -c spack -n '__fish_spack_using_command repo ls' -l namespaces -d 'show repository namespaces only' +complete -c spack -n '__fish_spack_using_command repo ls' -l json -f -a json +complete -c spack -n '__fish_spack_using_command repo ls' -l json -d 'output repositories as machine-readable json records' # spack repo add set -g __fish_spack_optspecs_spack_repo_add h/help name= path= scope= @@ -2941,6 +2957,18 @@ complete -c spack -n '__fish_spack_using_command repo update' -l tag -s t -r -d complete -c spack -n '__fish_spack_using_command repo update' -l commit -s c -r -f -a commit complete -c spack -n '__fish_spack_using_command repo update' -l commit -s c -r -d 'name of a commit to change to' +# spack repo show-version-updates +set -g __fish_spack_optspecs_spack_repo_show_version_updates h/help no-manual-packages no-git-versions only-redistributable + +complete -c spack -n '__fish_spack_using_command repo show-version-updates' -s h -l help -f -a help +complete -c spack -n '__fish_spack_using_command repo show-version-updates' -s h -l help -d 'show this help message and exit' +complete -c spack -n '__fish_spack_using_command repo show-version-updates' -l no-manual-packages -f -a no_manual_packages +complete -c spack -n '__fish_spack_using_command repo show-version-updates' -l no-manual-packages -d 'exclude manual packages' +complete -c spack -n '__fish_spack_using_command repo show-version-updates' -l no-git-versions -f -a no_git_versions +complete -c spack -n '__fish_spack_using_command repo show-version-updates' -l no-git-versions -d 'exclude versions from git' +complete -c spack -n '__fish_spack_using_command repo show-version-updates' -l only-redistributable -f -a only_redistributable +complete -c spack -n '__fish_spack_using_command repo show-version-updates' -l only-redistributable -d 'exclude non-redistributable packages' + # spack resource set -g __fish_spack_optspecs_spack_resource h/help complete -c spack -n '__fish_spack_using_command_pos 0 resource' -f -a list -d 'list all resources known to spack (currently just patches)' @@ -3081,7 +3109,7 @@ complete -c spack -n '__fish_spack_using_command style' -s h -l help -d 'show th complete -c spack -n '__fish_spack_using_command style' -s b -l base -r -f -a base complete -c spack -n '__fish_spack_using_command style' -s b -l base -r -d 'branch to compare against to determine changed files (default: develop)' complete -c spack -n '__fish_spack_using_command style' -s a -l all -f -a all -complete -c spack -n '__fish_spack_using_command style' -s a -l all -d 'check all files, not just changed files' +complete -c spack -n '__fish_spack_using_command style' -s a -l all -d 'check all files, not just changed files (applies only to Import Check)' complete -c spack -n '__fish_spack_using_command style' -s r -l root-relative -f -a root_relative complete -c spack -n '__fish_spack_using_command style' -s r -l root-relative -d 'print root-relative paths (default: cwd-relative)' complete -c spack -n '__fish_spack_using_command style' -s U -l no-untracked -f -a untracked @@ -3091,9 +3119,9 @@ complete -c spack -n '__fish_spack_using_command style' -s f -l fix -d 'format a complete -c spack -n '__fish_spack_using_command style' -l root -r -f -a root complete -c spack -n '__fish_spack_using_command style' -l root -r -d 'style check a different spack instance' complete -c spack -n '__fish_spack_using_command style' -s t -l tool -r -f -a tool -complete -c spack -n '__fish_spack_using_command style' -s t -l tool -r -d 'specify which tools to run (default: import, isort, black, flake8, mypy)' +complete -c spack -n '__fish_spack_using_command style' -s t -l tool -r -d 'specify which tools to run (default: import, ruff-format, ruff-check, mypy)' complete -c spack -n '__fish_spack_using_command style' -s s -l skip -r -f -a skip -complete -c spack -n '__fish_spack_using_command style' -s s -l skip -r -d 'specify tools to skip (choose from import, isort, black, flake8, mypy)' +complete -c spack -n '__fish_spack_using_command style' -s s -l skip -r -d 'specify tools to skip (choose from import, ruff-format, ruff-check, mypy)' complete -c spack -n '__fish_spack_using_command style' -l spec-strings -f -a spec_strings complete -c spack -n '__fish_spack_using_command style' -l spec-strings -d 'upgrade spec strings in Python, JSON and YAML files for compatibility with Spack v1.0 and v0.x. Example: spack style ``--spec-strings $(git ls-files)``. Note: must be used only on specs from spack v0.X.' @@ -3354,7 +3382,7 @@ complete -c spack -n '__fish_spack_using_command verify manifest' -s h -l help - complete -c spack -n '__fish_spack_using_command verify manifest' -s l -l local -f -a local complete -c spack -n '__fish_spack_using_command verify manifest' -s l -l local -d 'verify only locally installed packages' complete -c spack -n '__fish_spack_using_command verify manifest' -s j -l json -f -a json -complete -c spack -n '__fish_spack_using_command verify manifest' -s j -l json -d 'ouptut json-formatted errors' +complete -c spack -n '__fish_spack_using_command verify manifest' -s j -l json -d 'output json-formatted errors' complete -c spack -n '__fish_spack_using_command verify manifest' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command verify manifest' -s a -l all -d 'verify all packages' complete -c spack -n '__fish_spack_using_command verify manifest' -s s -l specs -f -a type diff --git a/share/spack/templates/bootstrap/spack.yaml b/share/spack/templates/bootstrap/spack.yaml index 8a178d03620459..6b8f0ca1a5e093 100644 --- a/share/spack/templates/bootstrap/spack.yaml +++ b/share/spack/templates/bootstrap/spack.yaml @@ -33,5 +33,15 @@ spack: require: "+wheel" concretizer: - reuse: false + reuse: true unify: true + targets: + granularity: generic + host_compatible: false + + mirrors:: +{% for mirror in bootstrap_mirrors %} + {{mirror}}: + url: https://binaries.spack.io/releases/v2026.03/{{mirror}} + signed: true +{% endfor %} diff --git a/share/spack/templates/modules/modulefile.tcl b/share/spack/templates/modules/modulefile.tcl index b162e3f62eb863..447ec9e0e75f31 100644 --- a/share/spack/templates/modules/modulefile.tcl +++ b/share/spack/templates/modules/modulefile.tcl @@ -27,15 +27,15 @@ proc ModulesHelp { } { {% block autoloads %} {% if autoload|length > 0 %} -if {![info exists ::env(LMOD_VERSION_MAJOR)]} { -{% for module in autoload %} - module load {{ module }} -{% endfor %} -} else { +# define missing command if using Environment Modules <5.1 +if {![llength [info commands depends-on]]} { + proc depends-on {args} { + module load {*}$args + } +} {% for module in autoload %} - depends-on {{ module }} +depends-on {{ module }} {% endfor %} -} {% endif %} {% endblock %} {# #} @@ -54,23 +54,11 @@ conflict {{ name }} {% block environment %} {% for command_name, cmd in environment_modifications %} {% if command_name == 'PrependPath' %} -{% if cmd.separator == ':' %} -prepend-path {{ cmd.name }} {{ '{' }}{{ cmd.value }}{{ '}' }} -{% else %} -prepend-path --delim {{ '{' }}{{ cmd.separator }}{{ '}' }} {{ cmd.name }} {{ '{' }}{{ cmd.value }}{{ '}' }} -{% endif %} +prepend-path -d {{ '{' }}{{ cmd.separator }}{{ '}' }} {{ cmd.name }} {{ '{' }}{{ cmd.value }}{{ '}' }} {% elif command_name in ('AppendPath', 'AppendFlagsEnv') %} -{% if cmd.separator == ':' %} -append-path {{ cmd.name }} {{ '{' }}{{ cmd.value }}{{ '}' }} -{% else %} -append-path --delim {{ '{' }}{{ cmd.separator }}{{ '}' }} {{ cmd.name }} {{ '{' }}{{ cmd.value }}{{ '}' }} -{% endif %} +append-path -d {{ '{' }}{{ cmd.separator }}{{ '}' }} {{ cmd.name }} {{ '{' }}{{ cmd.value }}{{ '}' }} {% elif command_name in ('RemovePath', 'RemoveFlagsEnv') %} -{% if cmd.separator == ':' %} -remove-path {{ cmd.name }} {{ '{' }}{{ cmd.value }}{{ '}' }} -{% else %} -remove-path --delim {{ '{' }}{{ cmd.separator }}{{ '}' }} {{ cmd.name }} {{ '{' }}{{ cmd.value }}{{ '}' }} -{% endif %} +remove-path -d {{ '{' }}{{ cmd.separator }}{{ '}' }} {{ cmd.name }} {{ '{' }}{{ cmd.value }}{{ '}' }} {% elif command_name == 'SetEnv' %} setenv {{ cmd.name }} {{ '{' }}{{ cmd.value }}{{ '}' }} {% elif command_name == 'UnsetEnv' %} diff --git a/var/spack/test_repos/spack_repo/builder_test/packages/inheritance_only_package/package.py b/var/spack/test_repos/spack_repo/builder_test/packages/inheritance_only_package/package.py new file mode 100644 index 00000000000000..b04dc67026df33 --- /dev/null +++ b/var/spack/test_repos/spack_repo/builder_test/packages/inheritance_only_package/package.py @@ -0,0 +1,12 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builder_test.packages.callbacks import package as callbacks + + +class InheritanceOnlyPackage(callbacks.Callbacks): + """Package used to verify that inheritance among packages works as expected, + when there is no override of the builder class. + """ + + pass diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/attributes_foo/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/attributes_foo/package.py index 75d7c359f902ba..23197eec546358 100644 --- a/var/spack/test_repos/spack_repo/builtin_mock/packages/attributes_foo/package.py +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/attributes_foo/package.py @@ -42,12 +42,12 @@ def headers(self): def libs(self): return find_libraries("libFoo", root=self.home, recursive=True) - # Header provided by the bar virutal package + # Header provided by the bar virtual package @property def bar_headers(self): return find_headers("bar", root=self.home.include, recursive=True) - # Libary provided by the bar virtual package + # Library provided by the bar virtual package @property def bar_libs(self): return find_libraries("libFooBar", root=self.home, recursive=True) diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/autotools_config_replacement/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/autotools_config_replacement/package.py index c042286982fe1b..5b27890ea2deab 100644 --- a/var/spack/test_repos/spack_repo/builtin_mock/packages/autotools_config_replacement/package.py +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/autotools_config_replacement/package.py @@ -47,7 +47,7 @@ def install(self, spec, prefix): def create_the_package_sources(self): # Creates the following file structure: # ./broken/config.sub -- not executable - # ./broken/config.guess -- exectuable & exit code 1 + # ./broken/config.guess -- executable & exit code 1 # ./working/config.sub -- executable & exit code 0 # ./working/config.guess -- executable & exit code 0 # Automatic config helper script substitution should replace the two @@ -70,7 +70,7 @@ def create_the_package_sources(self): with open(broken_config_sub, "w", encoding="utf-8") as f: f.write("#!/bin/sh\nexit 0") - # broken config.guess (exectuable but with error return code) + # broken config.guess (executable but with error return code) broken_config_guess = join_path(broken, "config.guess") with open(broken_config_guess, "w", encoding="utf-8") as f: f.write("#!/bin/sh\nexit 1") diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/binutils_for_test/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/binutils_for_test/package.py new file mode 100644 index 00000000000000..36626494005e8c --- /dev/null +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/binutils_for_test/package.py @@ -0,0 +1,21 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack_repo.builtin_mock.build_systems.generic import Package + +from spack.package import * + + +class BinutilsForTest(Package): + """A mock binutils-like package with a pure link dependency on zlib. + Used to test that transitive link-only deps of compiler run-deps are + not forced onto packages that use the compiler as a build dependency.""" + + homepage = "http://www.example.com" + url = "http://www.example.com/binutils-for-test-1.0.tar.gz" + + version("1.0", md5="0123456789abcdef0123456789abcdef") + + depends_on("c", type="build") + depends_on("zlib", type="link") diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/boost/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/boost/package.py index 66c21a34669322..e05d9c6766fc25 100644 --- a/var/spack/test_repos/spack_repo/builtin_mock/packages/boost/package.py +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/boost/package.py @@ -61,6 +61,6 @@ class Boost(Package): variant( "singlethreaded", default=False, description="Build single-threaded versions of libraries" ) - variant("icu", default=False, description="Build with Unicode and ICU suport") + variant("icu", default=False, description="Build with Unicode and ICU support") variant("graph", default=False, description="Build the Boost Graph library") variant("taggedlayout", default=False, description="Augment library names with build options") diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/compiler_with_deps/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/compiler_with_deps/package.py new file mode 100644 index 00000000000000..25bf648f2b2039 --- /dev/null +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/compiler_with_deps/package.py @@ -0,0 +1,31 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack_repo.builtin_mock.build_systems.compiler import CompilerPackage +from spack_repo.builtin_mock.build_systems.generic import Package + +from spack.package import * + + +class CompilerWithDeps(CompilerPackage, Package): + """A mock compiler that has a run+link dependency on binutils-for-test, + which itself has a pure link dependency on zlib. Used to test that + transitive link-only deps of compiler run-deps are not forced onto + packages that use this compiler as a build dependency.""" + + homepage = "http://www.example.com" + url = "http://www.example.com/compiler-with-deps-1.0.tar.gz" + + version("1.0", md5="0123456789abcdef0123456789abcdef") + + provides("c") + + depends_on("c", type="build") + depends_on("binutils-for-test", type=("run", "link")) + + c_names = ["compiler-with-deps-cc"] + compiler_version_regex = r"([0-9.]+)" + compiler_version_argument = "--version" + + compiler_wrapper_link_paths = {"c": "compiler-with-deps/cc"} diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/conditionally_patch_dependency/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/conditionally_patch_dependency/package.py index 3c7df9fb283f9c..3fd2322aa5719d 100644 --- a/var/spack/test_repos/spack_repo/builtin_mock/packages/conditionally_patch_dependency/package.py +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/conditionally_patch_dependency/package.py @@ -9,7 +9,7 @@ class ConditionallyPatchDependency(Package): - """Package that conditionally requries a patched version + """Package that conditionally requires a patched version of a dependency.""" homepage = "http://www.example.com" diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/conflict_virtual/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/conflict_virtual/package.py new file mode 100644 index 00000000000000..94946e13581869 --- /dev/null +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/conflict_virtual/package.py @@ -0,0 +1,16 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack.package import * + + +class ConflictVirtual(Package): + version("1.0") + variant("conflict_direct", default=False, description="Enable conflict") + variant("conflict_transitive", default=False, description="Enable conflict") + + depends_on("blas") + requires("%blas=netlib-blas") + + conflicts("%blas=netlib-blas", when="+conflict_direct") + conflicts("^blas=netlib-blas", when="+conflict_transitive") diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/direct_dep_virtuals_one/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/direct_dep_virtuals_one/package.py new file mode 100644 index 00000000000000..34fe4c36b5a0b5 --- /dev/null +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/direct_dep_virtuals_one/package.py @@ -0,0 +1,14 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack.package import * + + +class DirectDepVirtualsOne(Package): + version("1.0") + # These two statements imply that %blas=netlib-blas must be false. + depends_on("direct-dep-virtuals-two +variant", when="%blas=netlib-blas") + depends_on("direct-dep-virtuals-two ~variant") + + # The provider is a direct dependency, but its virtual is *not* depended on. + depends_on("netlib-blas") diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/direct_dep_virtuals_two/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/direct_dep_virtuals_two/package.py new file mode 100644 index 00000000000000..e8ad566e523ce3 --- /dev/null +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/direct_dep_virtuals_two/package.py @@ -0,0 +1,13 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack.package import * + + +class DirectDepVirtualsTwo(Package): + version("1.0") + variant("variant", default=False) + # Pick netlib-blas as a provider for blas. + depends_on("blas") + # Require that netlib-blas is a dependency (and thus the provider of blas). + requires("%netlib-blas") diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/fail_test_audit_deprecated/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/fail_test_audit_deprecated/package.py deleted file mode 100644 index d448fd65defbc9..00000000000000 --- a/var/spack/test_repos/spack_repo/builtin_mock/packages/fail_test_audit_deprecated/package.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright Spack Project Developers. See COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack_repo.builtin_mock.build_systems.makefile import MakefilePackage - -from spack.package import * - - -class FailTestAuditDeprecated(MakefilePackage): - """Simple package attempting to implement and use deprecated stand-alone test methods.""" - - homepage = "http://github.com/dummy/fail-test-audit-deprecated" - url = "https://github.com/dummy/fail-test-audit-deprecated/archive/v1.0.tar.gz" - - version("2.0", sha256="c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1") - version("1.0", sha256="abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234") - - @run_after("install") - def copy_test_files(self): - """test that uses the deprecated install_test_root method""" - self.cache_extra_test_sources(".") - - def test(self): - """this is a deprecated reserved method for stand-alone testing""" - pass - - def test_use_install_test_root(self): - """use the deprecated install_test_root method""" - print(f"install test root = {self.install_test_root()}") - - def test_run_test(self): - """use the deprecated run_test method""" - self.run_test("which", ["make"]) diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/gcc/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/gcc/package.py index a433f6d403b611..b2374251e006fc 100644 --- a/var/spack/test_repos/spack_repo/builtin_mock/packages/gcc/package.py +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/gcc/package.py @@ -36,6 +36,9 @@ class Gcc(CompilerPackage, Package): description="Compilers and runtime libraries to build", ) + # This variant is here so that we can test having externals using the non-default value + variant("binutils", default=True, description="") + provides("c", "cxx", when="languages=c,c++") provides("c", when="languages=c") provides("cxx", when="languages=c++") diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/invalid_maintainer/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/invalid_maintainer/package.py new file mode 100644 index 00000000000000..ed653dc1d60da6 --- /dev/null +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/invalid_maintainer/package.py @@ -0,0 +1,14 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack.package import * + + +class InvalidMaintainer(Package): + """Package with invalid maintainers (placeholders).""" + + url = "https://www.invalid-maintainer.org/archive/v1.0.tar.gz" + + maintainers("github_user1", "github_user2") + + version("1.0", sha256="0f22de2391d80d8b393c4f9d11488600126c60ae36ceef780c6a4b3d9dab2e96") diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/llvm/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/llvm/package.py index eb47f5f847c34a..acf6067b6b2034 100644 --- a/var/spack/test_repos/spack_repo/builtin_mock/packages/llvm/package.py +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/llvm/package.py @@ -33,6 +33,7 @@ class Llvm(Package, CompilerPackage): provides("c", "cxx", when="+clang") provides("fortran", when="+flang") + provides("libllvm") depends_on("c") diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/mesa/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/mesa/package.py new file mode 100644 index 00000000000000..fefac7ee208ae9 --- /dev/null +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/mesa/package.py @@ -0,0 +1,14 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack.package import * + + +class Mesa(Package): + """Package depending on libllvm (a link-type virtual provided by a compiler)""" + + homepage = "https://www.mesa.com" + + version("2.0.1") + depends_on("libllvm") + depends_on("cxx", type="build") diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/mixing_parent/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/mixing_parent/package.py index cce56c8ad5932b..7b2ccb35d6b05d 100644 --- a/var/spack/test_repos/spack_repo/builtin_mock/packages/mixing_parent/package.py +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/mixing_parent/package.py @@ -6,7 +6,6 @@ class MixingParent(Package): - homepage = "http://www.example.com" url = "http://www.example.com/a-1.0.tar.gz" diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/multivalue_variant_multi_defaults/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/multivalue_variant_multi_defaults/package.py new file mode 100644 index 00000000000000..ef18eabb8a4ea5 --- /dev/null +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/multivalue_variant_multi_defaults/package.py @@ -0,0 +1,25 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack_repo.builtin_mock.build_systems.generic import Package + +from spack.package import * + + +class MultivalueVariantMultiDefaults(Package): + homepage = "http://www.spack.llnl.gov" + url = "http://www.spack.llnl.gov/mpileaks-1.0.tar.gz" + + version("1.0", md5="0123456789abcdef0123456789abcdef") + + variant( + "myvariant", + default="bar,baz", + values=("bar", "baz"), + multi=True, + description="Type of libraries to install", + ) + + # conditional dep to incur a cost for packages to build when myvariant includes baz + depends_on("trivial-install-test-package", when="myvariant=baz") diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/multivalue_variant_multi_defaults_dependent/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/multivalue_variant_multi_defaults_dependent/package.py new file mode 100644 index 00000000000000..baf8c7aa9a789d --- /dev/null +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/multivalue_variant_multi_defaults_dependent/package.py @@ -0,0 +1,18 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack_repo.builtin_mock.build_systems.generic import Package + +from spack.package import * + + +class MultivalueVariantMultiDefaultsDependent(Package): + homepage = "http://www.spack.llnl.gov" + url = "http://www.spack.llnl.gov/mpileaks-1.0.tar.gz" + + version("1.0", md5="0123456789abcdef0123456789abcdef") + + # includes a subset of the default values `bar,baz`; we expect the request for myvariant=bar + # not to override the default myvariant=bar,baz + depends_on("multivalue-variant-multi-defaults myvariant=bar") diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/paraview/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/paraview/package.py new file mode 100644 index 00000000000000..7b32f31f7f354e --- /dev/null +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/paraview/package.py @@ -0,0 +1,19 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack_repo.builtin_mock.build_systems.generic import Package + +from spack.package import * + + +class Paraview(Package): + """Package depending on a library, that has a link dependency to libllvm""" + + homepage = "http://www.example.com" + url = "http://www.example.com/c-1.0.tar.gz" + + version("1.0", md5="0123456789abcdef0123456789abcdef") + + depends_on("mesa") + depends_on("cxx", type="build") diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/patch/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/patch/package.py index d02c8a5f84c58c..7ab97dd06e428a 100644 --- a/var/spack/test_repos/spack_repo/builtin_mock/packages/patch/package.py +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/patch/package.py @@ -8,7 +8,7 @@ class Patch(Package): - """Package that requries a patched version of a dependency.""" + """Package that requires a patched version of a dependency.""" homepage = "http://www.example.com" url = "http://www.example.com/patch-1.0.tar.gz" diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/patch_a_dependency/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/patch_a_dependency/package.py index 8960d60a680d3d..db3113c0d6348d 100644 --- a/var/spack/test_repos/spack_repo/builtin_mock/packages/patch_a_dependency/package.py +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/patch_a_dependency/package.py @@ -8,7 +8,7 @@ class PatchADependency(Package): - """Package that requries a patched version of a dependency.""" + """Package that requires a patched version of a dependency.""" homepage = "http://www.example.com" url = "http://www.example.com/patch-a-dependency-1.0.tar.gz" diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/patch_several_dependencies/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/patch_several_dependencies/package.py index ded1209b307178..978db4841e6d07 100644 --- a/var/spack/test_repos/spack_repo/builtin_mock/packages/patch_several_dependencies/package.py +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/patch_several_dependencies/package.py @@ -8,7 +8,7 @@ class PatchSeveralDependencies(Package): - """Package that requries multiple patches on a dependency.""" + """Package that requires multiple patches on a dependency.""" homepage = "http://www.example.com" url = "http://www.example.com/patch-a-dependency-1.0.tar.gz" diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/pkg_with_c_link_dep/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/pkg_with_c_link_dep/package.py new file mode 100644 index 00000000000000..3ff6657415eecf --- /dev/null +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/pkg_with_c_link_dep/package.py @@ -0,0 +1,16 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack.package import * + + +class PkgWithCLinkDep(Package): + """Simple package with one optional dependency""" + + homepage = "http://www.example.com" + url = "http://www.example.com/a-1.0.tar.gz" + + version("1.0", md5="0123456789abcdef0123456789abcdef") + + # This package erroneously declares a build,link dependency on c + depends_on("c") diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/pkg_with_zlib_dep/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/pkg_with_zlib_dep/package.py new file mode 100644 index 00000000000000..4e05eb43ad897a --- /dev/null +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/pkg_with_zlib_dep/package.py @@ -0,0 +1,22 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack_repo.builtin_mock.build_systems.generic import Package + +from spack.package import * + + +class PkgWithZlibDep(Package): + """A minimal mock package that depends on C (build) and zlib (link). + Used to test that the compiler's transitive link-only deps (reachable + through its run-dep binutils-for-test -> zlib) are not forced onto + this package's own zlib dependency.""" + + homepage = "http://www.example.com" + url = "http://www.example.com/pkg-with-zlib-dep-1.0.tar.gz" + + version("1.0", md5="0123456789abcdef0123456789abcdef") + + depends_on("c", type="build") + depends_on("zlib", type="link") diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/placeholder/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/placeholder/package.py new file mode 100644 index 00000000000000..c02c79acbf837f --- /dev/null +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/placeholder/package.py @@ -0,0 +1,22 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack_repo.builtin_mock.build_systems.generic import Package + +from spack.package import * + + +class Placeholder(Package): + """Placeholder test package""" + + version("1.5") + + @property + def fetcher(self): + msg = "Placeholder package" + raise InstallError(msg) + + @fetcher.setter + def fetcher(self, value): + _ = self.fetcher diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/flake8/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/ruff/package.py similarity index 59% rename from var/spack/test_repos/spack_repo/builtin_mock/packages/flake8/package.py rename to var/spack/test_repos/spack_repo/builtin_mock/packages/ruff/package.py index 15dc5c69d19bfc..d86eea9400d527 100644 --- a/var/spack/test_repos/spack_repo/builtin_mock/packages/flake8/package.py +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/ruff/package.py @@ -7,14 +7,14 @@ from spack.package import * -class Flake8(Package): - """Package containing as many acceptable ``PEP8`` violations as possible. +class Ruff(Package): + """Package containing ``PEP8`` violations. - All of these violations are exceptions that we allow in ``package.py`` files, and - Spack is more lenient than ``flake8`` is for things like URLs and long SHA sums. + Ruff check + format handle most errors robustly and those that + cannot be handled directly are infrequent enough we can noqa them - See ``share/spack/qa/flake8_formatter.py`` for specifics of how we handle ``flake8`` - exemptions. + This file contains a number of errors ruff should be able to reformat + and pass style over """ @@ -35,31 +35,25 @@ class Flake8(Package): # All URL strings are exempt from line-length checks. # - # flake8 normally would complain about these, but the fix it wants (a multi-line - # string) is ugbly, and we're more lenient since there are many places where Spack - # wants URLs in strings. - hg = "https://example.com/this-is-a-really-long-url/that-goes-over-99-characters/that-flake8-will-not-ignore-by-default" - list_url = "https://example.com/this-is-a-really-long-url/that-goes-over-99-characters/that-flake8-will-not-ignore-by-default" - git = "ssh://example.com/this-is-a-really-long-url/that-goes-over-99-characters/that-flake8-will-not-ignore-by-default" + # ruff will not complain about these + hg = "https://example.com/this-is-a-really-long-url/that-goes-over-99-characters/that-ruff-will-not-ignore-by-default" + list_url = "https://example.com/this-is-a-really-long-url/that-goes-over-99-characters/that-ruff-will-not-ignore-by-default" + git = "ssh://example.com/this-is-a-really-long-url/that-goes-over-99-characters/that-ruff-will-not-ignore-by-default" # directives with URLs are exempt as well version( "1.0", - url="https://example.com/this-is-a-really-long-url/that-goes-over-99-characters/that-flake8-will-not-ignore-by-default", + url="https://example.com/this-is-a-really-long-url/that-goes-over-99-characters/that-ruff-will-not-ignore-by-default", ) # - # Also test URL comments (though flake8 will ignore these by default anyway) + # Also test URL comments (though ruff will ignore these by default anyway) # - # http://example.com/this-is-a-really-long-url/that-goes-over-99-characters/that-flake8-will-ignore-by-default - # https://example.com/this-is-a-really-long-url/that-goes-over-99-characters/that-flake8-will-ignore-by-default - # ftp://example.com/this-is-a-really-long-url/that-goes-over-99-characters/that-flake8-will-ignore-by-default - # ssh://example.com/this-is-a-really-long-url/that-goes-over-99-characters/that-flake8-will-ignore-by-default - # file://example.com/this-is-a-really-long-url/that-goes-over-99-characters/that-flake8-will-ignore-by-default - - # Strings and comments with really long checksums require no noqa annotation. - sha512sum = "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" - # the sha512sum is "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" + # http://example.com/this-is-a-really-long-url/that-goes-over-99-characters/that-ruff-will-ignore-by-default + # https://example.com/this-is-a-really-long-url/that-goes-over-99-characters/that-ruff-will-ignore-by-default + # ftp://example.com/this-is-a-really-long-url/that-goes-over-99-characters/that-ruff-will-ignore-by-default + # ssh://example.com/this-is-a-really-long-url/that-goes-over-99-characters/that-ruff-will-ignore-by-default + # file://example.com/this-is-a-really-long-url/that-goes-over-99-characters/that-ruff-will-ignore-by-default def install(self, spec, prefix): # Make sure lines with '# noqa' work as expected. Don't just diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/single_language_virtual/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/single_language_virtual/package.py new file mode 100644 index 00000000000000..451de0bf489976 --- /dev/null +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/single_language_virtual/package.py @@ -0,0 +1,24 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack_repo.builtin_mock.build_systems.generic import Package + +from spack.package import * + + +class SingleLanguageVirtual(Package): + """Package using a single language virtual for compilation""" + + homepage = "http://www.example.com" + url = "http://www.example.com/foo-1.0.tar.gz" + + version("1.0", md5="0123456789abcdef0123456789abcdef") + + variant("c", default=False) + variant("cxx", default=False) + variant("fortran", default=False) + + depends_on("c", when="+c") + depends_on("cxx", when="+cxx") + depends_on("fortran", when="+fortran") diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/variant_function_validator/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/variant_function_validator/package.py new file mode 100644 index 00000000000000..a07f131842bd98 --- /dev/null +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/variant_function_validator/package.py @@ -0,0 +1,27 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin_mock.build_systems.generic import Package + +from spack.package import * + + +def _allowed_values(x): + return x in {"make", "ninja", "other"} + + +class VariantFunctionValidator(Package): + """This package has a variant with values defined by a function validator.""" + + homepage = "https://www.example.org" + url = "https://example.org/files/v3.4/cmake-3.4.3.tar.gz" + + version("1.0", md5="4cb3ff35b2472aae70f542116d616e63") + + variant("generator", default="make", values=_allowed_values, description="?") + + # Create a situation where, if the penalty for the variant defined by a function + # is not taken into account, then we'll select the non-default value + depends_on("adios2") + conflicts("adios2+bzip2", when="generator=make") + conflicts("adios2~bzip2", when="generator=ninja") diff --git a/var/spack/test_repos/spack_repo/duplicates_test/packages/py_numpy/package.py b/var/spack/test_repos/spack_repo/duplicates_test/packages/py_numpy/package.py index f668c97cfc0dd8..3f99f3068433b1 100644 --- a/var/spack/test_repos/spack_repo/duplicates_test/packages/py_numpy/package.py +++ b/var/spack/test_repos/spack_repo/duplicates_test/packages/py_numpy/package.py @@ -17,5 +17,5 @@ class PyNumpy(Package): version("1.25.0", md5="0123456789abcdef0123456789abcdef") extends("python") - depends_on("py-setuptools@=59", type=("build", "run")) + depends_on("py-setuptools@=59", type="build") depends_on("gmake@4.1", type="build") diff --git a/var/spack/test_repos/spack_repo/duplicates_test/packages/unify_build_deps_a/package.py b/var/spack/test_repos/spack_repo/duplicates_test/packages/unify_build_deps_a/package.py new file mode 100644 index 00000000000000..35cd84fd15dbe5 --- /dev/null +++ b/var/spack/test_repos/spack_repo/duplicates_test/packages/unify_build_deps_a/package.py @@ -0,0 +1,22 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack.package import * + + +class UnifyBuildDepsA(Package): + """Used to test that we cannot have a build environment with two conflicting versions of + a package (unify-build-deps-c), even if that package is tagged as a build-tool with duplicates + allowed.""" + + url = "http://example.com/unify-build-deps-a-1.0.tar.gz" + version("2.0", sha256="d41d8cd98f00b204e9800998ecf8427ed41d8cd98f00b204e9800998ecf8427e") + version("1.0", sha256="d41d8cd98f00b204e9800998ecf8427ed41d8cd98f00b204e9800998ecf8427e") + + depends_on("unify-build-deps-c@1", type="build") + + # If unify-build-deps-b is used as a build dependency, we cannot unify the build environment. + depends_on("unify-build-deps-b", type=("build", "run"), when="@1") + + # If unify-build-deps-b is not used as build dependency, we can unify the build environment + depends_on("unify-build-deps-b", type=("link", "run"), when="@2") diff --git a/var/spack/test_repos/spack_repo/duplicates_test/packages/unify_build_deps_b/package.py b/var/spack/test_repos/spack_repo/duplicates_test/packages/unify_build_deps_b/package.py new file mode 100644 index 00000000000000..1add87feb72470 --- /dev/null +++ b/var/spack/test_repos/spack_repo/duplicates_test/packages/unify_build_deps_b/package.py @@ -0,0 +1,11 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack.package import * + + +class UnifyBuildDepsB(Package): + url = "http://example.com/unify-build-deps-b-1.0.tar.gz" + version("1.0", sha256="d41d8cd98f00b204e9800998ecf8427ed41d8cd98f00b204e9800998ecf8427e") + + depends_on("unify-build-deps-c@2", type="run") diff --git a/var/spack/test_repos/spack_repo/duplicates_test/packages/unify_build_deps_c/package.py b/var/spack/test_repos/spack_repo/duplicates_test/packages/unify_build_deps_c/package.py new file mode 100644 index 00000000000000..9214f8c10efdf9 --- /dev/null +++ b/var/spack/test_repos/spack_repo/duplicates_test/packages/unify_build_deps_c/package.py @@ -0,0 +1,11 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack.package import * + + +class UnifyBuildDepsC(Package): + tags = ["build-tools"] + url = "http://example.com/unify-build-deps-c-1.0.tar.gz" + version("2.0", sha256="d41d8cd98f00b204e9800998ecf8427ed41d8cd98f00b204e9800998ecf8427e") + version("1.0", sha256="d41d8cd98f00b204e9800998ecf8427ed41d8cd98f00b204e9800998ecf8427e") diff --git a/var/spack/test_repos/spack_repo/tutorial/packages/hdf5/package.py b/var/spack/test_repos/spack_repo/tutorial/packages/hdf5/package.py index d18a8c5006020f..e00ac180aedcef 100644 --- a/var/spack/test_repos/spack_repo/tutorial/packages/hdf5/package.py +++ b/var/spack/test_repos/spack_repo/tutorial/packages/hdf5/package.py @@ -269,7 +269,7 @@ def libs(self): # starting version 1.8.10) does not produce it. Instead, the # basename of the library file is 'libhdf5_hl_fortran'. Which # means that switching to CMake requires rebuilding of all - # dependant packages that use the High-level Fortran interface. + # dependent packages that use the High-level Fortran interface. # Therefore, we do not try to preserve backward compatibility # with Autotools installations by creating symlinks. The only # packages that could benefit from it would be those that @@ -331,7 +331,8 @@ def cmake_args(self): self.define("HDF5_BUILD_EXAMPLES", False), self.define( "BUILD_TESTING", - self.run_tests or + self.run_tests + or # Version 1.8.22 fails to build the tools when shared libraries # are enabled but the tests are disabled. spec.satisfies("@1.8.22+shared+tools"), @@ -386,9 +387,7 @@ def ensure_parallel_compiler_wrappers(self): # 1.10.6 and 1.12.0. The current develop versions do not produce 'h5pfc' # at all. Here, we make sure that 'h5pfc' is available when Fortran and # MPI support are enabled (only for versions that generate 'h5fc'). - if self.spec.satisfies( - "@1.8.22:1.8," "1.10.6:1.10," "1.12.0:1.12," "develop:" "+fortran+mpi" - ): + if self.spec.satisfies("@1.8.22:1.8,1.10.6:1.10,1.12.0:1.12,develop:+fortran+mpi"): with working_dir(self.prefix.bin): # No try/except here, fix the condition above instead: symlink("h5fc", "h5pfc") @@ -458,9 +457,7 @@ def _check_install(self): """ expected = """\ HDF5 version {version} {version} -""".format( - version=str(spec.version.up_to(3)) - ) +""".format(version=str(spec.version.up_to(3))) with open("check.c", "w", encoding="utf-8") as f: f.write(source) if "+mpi" in spec: @@ -536,7 +533,7 @@ def _test_example(self): "h5copy", options, [], installed=True, purpose=reason, skip_missing=True, work_dir="." ) - reason = "test: ensuring h5diff shows no differences between orig and" " copy" + reason = "test: ensuring h5diff shows no differences between orig and copy" self.run_test( "h5diff", [h5_file, "test.h5"],