diff --git a/.gitattributes b/.gitattributes index 0dac0f84927119..b8189f12ded0f4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -112,3 +112,4 @@ Tools/peg_generator/pegen/grammar_parser.py generated aclocal.m4 generated configure generated *.min.js generated +package-lock.json generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 63ec65ac0dbd8d..c5348d606b82d8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -427,19 +427,19 @@ Lib/dataclasses.py @ericvsmith Lib/test/test_dataclasses/ @ericvsmith # Dates and times -Doc/**/*time.rst @pganssle @abalkin @StanFromIreland +Doc/**/*time.rst @pganssle @StanFromIreland Doc/library/datetime-* @pganssle @StanFromIreland Doc/library/zoneinfo.rst @pganssle @StanFromIreland -Include/datetime.h @pganssle @abalkin @StanFromIreland -Include/internal/pycore_time.h @pganssle @abalkin @StanFromIreland +Include/datetime.h @pganssle @StanFromIreland +Include/internal/pycore_time.h @pganssle @StanFromIreland Lib/test/test_zoneinfo/ @pganssle @StanFromIreland Lib/zoneinfo/ @pganssle @StanFromIreland -Lib/*time.py @pganssle @abalkin @StanFromIreland -Lib/test/datetimetester.py @pganssle @abalkin @StanFromIreland -Lib/test/test_*time.py @pganssle @abalkin @StanFromIreland +Lib/*time.py @pganssle @StanFromIreland +Lib/test/datetimetester.py @pganssle @StanFromIreland +Lib/test/test_*time.py @pganssle @StanFromIreland Modules/*zoneinfo* @pganssle @StanFromIreland -Modules/*time* @pganssle @abalkin @StanFromIreland -Python/pytime.c @pganssle @abalkin @StanFromIreland +Modules/*time* @pganssle @StanFromIreland +Python/pytime.c @pganssle @StanFromIreland # Dbm Doc/library/dbm.rst @corona10 @erlend-aasland @serhiy-storchaka diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e68a07382d5884..4b77646e22db4b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,7 +3,7 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "monthly" + interval: "quarterly" labels: - "skip issue" - "skip news" @@ -24,7 +24,7 @@ updates: - package-ecosystem: "pip" directory: "/Tools/" schedule: - interval: "monthly" + interval: "quarterly" labels: - "skip issue" - "skip news" diff --git a/.github/workflows/add-issue-header.yml b/.github/workflows/add-issue-header.yml index c404bc519300e2..00b7ae50cb9935 100644 --- a/.github/workflows/add-issue-header.yml +++ b/.github/workflows/add-issue-header.yml @@ -12,6 +12,7 @@ on: # Only ever run once - opened +permissions: {} jobs: add-header: @@ -20,7 +21,7 @@ jobs: issues: write timeout-minutes: 5 steps: - - uses: actions/github-script@v8 + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: # language=JavaScript script: | diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2fa2ab768dc48b..c34f8f699d8edb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,8 +11,7 @@ on: - 'main' - '3.*' -permissions: - contents: read +permissions: {} concurrency: # https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#concurrency @@ -64,7 +63,7 @@ jobs: run: | apt update && apt install git -yq git config --global --add safe.directory "$GITHUB_WORKSPACE" - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 1 persist-credentials: false @@ -101,10 +100,10 @@ jobs: needs: build-context if: needs.build-context.outputs.run-tests == 'true' steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.x' - name: Runner image version @@ -165,13 +164,21 @@ jobs: free-threading: - false - true + interpreter: + - switch-case exclude: # Skip Win32 on free-threaded builds - { arch: Win32, free-threading: true } + include: + # msvc::musttail is currently only supported on x64, + # and only supported on 3.15+. + - { arch: x64, free-threading: false, interpreter: tail-call } + - { arch: x64, free-threading: true, interpreter: tail-call } uses: ./.github/workflows/reusable-windows.yml with: arch: ${{ matrix.arch }} free-threading: ${{ matrix.free-threading }} + interpreter: ${{ matrix.interpreter }} build-windows-msi: # ${{ '' } is a hack to nest jobs under the same sidebar category. @@ -198,10 +205,10 @@ jobs: strategy: fail-fast: false matrix: - # macos-14 is M1, macos-15-intel is Intel. + # macos-26 is Apple Silicon, macos-15-intel is Intel. # macos-15-intel only runs tests against the GIL-enabled CPython. os: - - macos-14 + - macos-26 - macos-15-intel free-threading: - false @@ -283,7 +290,7 @@ jobs: SSLLIB_DIR: ${{ github.workspace }}/multissl/${{ matrix.ssllib.name }}/${{ matrix.ssllib.version }} LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/${{ matrix.ssllib.name }}/${{ matrix.ssllib.version }}/lib steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Runner image version @@ -294,7 +301,7 @@ jobs: run: sudo ./.github/workflows/posix-deps-apt.sh - name: 'Restore SSL library build' id: cache-ssl-lib - uses: actions/cache@v5 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ./multissl/${{ matrix.ssllib.name }}/${{ matrix.ssllib.version }} key: ${{ matrix.os }}-multissl-${{ matrix.ssllib.name }}-${{ matrix.ssllib.version }} @@ -336,13 +343,13 @@ jobs: matrix: include: - arch: aarch64 - runs-on: macos-14 + runs-on: macos-26 - arch: x86_64 runs-on: ubuntu-24.04 runs-on: ${{ matrix.runs-on }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Build and test @@ -355,7 +362,7 @@ jobs: timeout-minutes: 60 runs-on: macos-14 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false @@ -369,7 +376,13 @@ jobs: sudo xcode-select --switch /Applications/Xcode_15.4.app - name: Build and test - run: python3 Apple ci iOS --fast-ci --simulator 'iPhone SE (3rd generation),OS=17.5' + run: python3 Platforms/Apple ci iOS --fast-ci --simulator 'iPhone SE (3rd generation),OS=17.5' + + build-emscripten: + name: 'Emscripten' + needs: build-context + if: needs.build-context.outputs.run-emscripten == 'true' + uses: ./.github/workflows/reusable-emscripten.yml build-wasi: name: 'WASI' @@ -387,7 +400,7 @@ jobs: OPENSSL_VER: 3.5.5 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Register gcc problem matcher @@ -401,7 +414,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v5 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -448,7 +461,7 @@ jobs: ./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt" - name: 'Restore Hypothesis database' id: cache-hypothesis-database - uses: actions/cache@v5 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ key: hypothesis-database-${{ github.head_ref || github.run_id }} @@ -475,7 +488,7 @@ jobs: -x test_subprocess \ -x test_signal \ -x test_sysconfig - - uses: actions/upload-artifact@v7 + - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 if: always() with: name: hypothesis-example-db @@ -496,7 +509,7 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Runner image version @@ -506,7 +519,7 @@ jobs: - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Set up GCC-10 for ASAN - uses: egor-tensin/setup-gcc@v2 + uses: egor-tensin/setup-gcc@a2861a8b8538f49cf2850980acccf6b05a1b2ae4 # v2.0 with: version: 10 - name: Configure OpenSSL env vars @@ -516,7 +529,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v5 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -563,7 +576,7 @@ jobs: needs: build-context if: needs.build-context.outputs.run-ubuntu == 'true' steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Runner image version @@ -650,6 +663,7 @@ jobs: - build-ubuntu - build-ubuntu-ssltests - build-ios + - build-emscripten - build-wasi - test-hypothesis - build-asan @@ -664,6 +678,7 @@ jobs: with: allowed-failures: >- build-android, + build-emscripten, build-windows-msi, build-ubuntu-ssltests, test-hypothesis, @@ -706,5 +721,6 @@ jobs: }} ${{ !fromJSON(needs.build-context.outputs.run-android) && 'build-android,' || '' }} ${{ !fromJSON(needs.build-context.outputs.run-ios) && 'build-ios,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-emscripten) && 'build-emscripten,' || '' }} ${{ !fromJSON(needs.build-context.outputs.run-wasi) && 'build-wasi,' || '' }} jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml index a09a30587b35eb..19314dd0c889b0 100644 --- a/.github/workflows/documentation-links.yml +++ b/.github/workflows/documentation-links.yml @@ -22,7 +22,7 @@ jobs: timeout-minutes: 5 steps: - - uses: readthedocs/actions/preview@v1 + - uses: readthedocs/actions/preview@b8bba1484329bda1a3abe986df7ebc80a8950333 # v1.5 with: project-slug: "cpython-previews" single-version: "true" diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index da9c75ec75391a..81d75ef1820903 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -15,8 +15,7 @@ on: paths: *paths workflow_dispatch: -permissions: - contents: read +permissions: {} concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -32,7 +31,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Build tier two interpreter @@ -69,10 +68,10 @@ jobs: architecture: ARM64 runner: windows-11-arm steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' # PCbuild downloads LLVM automatically: @@ -101,12 +100,12 @@ jobs: - target: x86_64-apple-darwin/clang runner: macos-15-intel - target: aarch64-apple-darwin/clang - runner: macos-14 + runner: macos-15 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' - name: Install LLVM @@ -146,10 +145,10 @@ jobs: - target: aarch64-unknown-linux-gnu/gcc runner: ubuntu-24.04-arm steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' - name: Build @@ -182,10 +181,10 @@ jobs: use_clang: true run_tests: false steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' - name: Build diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 0ded53b00da0ef..fb2b94b7362308 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,8 +2,7 @@ name: Lint on: [push, pull_request, workflow_dispatch] -permissions: - contents: read +permissions: {} env: FORCE_COLOR: 1 @@ -19,7 +18,7 @@ jobs: timeout-minutes: 10 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: j178/prek-action@v1 + - uses: j178/prek-action@0bb87d7f00b0c99306c8bcb8b8beba1eb581c037 # v1.1.1 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index db363bef7a45ae..583dc1808dfc35 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -33,8 +33,7 @@ on: - "Tools/requirements-dev.txt" workflow_dispatch: -permissions: - contents: read +permissions: {} env: PIP_DISABLE_PIP_VERSION_CHECK: 1 @@ -65,10 +64,10 @@ jobs: "Tools/peg_generator", ] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.13" cache: pip diff --git a/.github/workflows/new-bugs-announce-notifier.yml b/.github/workflows/new-bugs-announce-notifier.yml index 9ee38a4fd1cefc..be375a970a475c 100644 --- a/.github/workflows/new-bugs-announce-notifier.yml +++ b/.github/workflows/new-bugs-announce-notifier.yml @@ -5,20 +5,21 @@ on: types: - opened -permissions: - issues: read +permissions: {} jobs: notify-new-bugs-announce: runs-on: ubuntu-latest + permissions: + issues: read timeout-minutes: 10 steps: - - uses: actions/setup-node@v6 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: 20 - run: npm install mailgun.js form-data - name: Send notification - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: MAILGUN_API_KEY: ${{ secrets.MAILGUN_PYTHON_ORG_MAILGUN_KEY }} with: diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index 7e534c58c798d1..262299fc30f989 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -4,6 +4,8 @@ on: pull_request: types: [opened, reopened, labeled, unlabeled, synchronize] +permissions: {} + jobs: label-dnm: name: DO-NOT-MERGE @@ -15,7 +17,7 @@ jobs: steps: - name: Check there's no DO-NOT-MERGE - uses: mheap/github-action-required-labels@v5 + uses: mheap/github-action-required-labels@0ac283b4e65c1fb28ce6079dea5546ceca98ccbe # v5.5.2 with: mode: exactly count: 0 @@ -33,7 +35,7 @@ jobs: steps: # Check that the PR is not awaiting changes from the author due to previous review. - name: Check there's no required changes - uses: mheap/github-action-required-labels@v5 + uses: mheap/github-action-required-labels@0ac283b4e65c1fb28ce6079dea5546ceca98ccbe # v5.5.2 with: mode: exactly count: 0 @@ -42,7 +44,7 @@ jobs: awaiting change review - id: is-feature name: Check whether this PR is a feature (contains a "type-feature" label) - uses: mheap/github-action-required-labels@v5 + uses: mheap/github-action-required-labels@0ac283b4e65c1fb28ce6079dea5546ceca98ccbe # v5.5.2 with: mode: exactly count: 1 @@ -53,7 +55,7 @@ jobs: - id: awaiting-merge if: steps.is-feature.outputs.status == 'success' name: Check for complete review - uses: mheap/github-action-required-labels@v5 + uses: mheap/github-action-required-labels@0ac283b4e65c1fb28ce6079dea5546ceca98ccbe # v5.5.2 with: mode: exactly count: 1 diff --git a/.github/workflows/reusable-check-c-api-docs.yml b/.github/workflows/reusable-check-c-api-docs.yml index b95bd6a0184ea7..5fae57a1dbda36 100644 --- a/.github/workflows/reusable-check-c-api-docs.yml +++ b/.github/workflows/reusable-check-c-api-docs.yml @@ -3,8 +3,7 @@ name: Reusable C API Docs Check on: workflow_call: -permissions: - contents: read +permissions: {} env: FORCE_COLOR: 1 @@ -15,10 +14,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.x' - name: Check for undocumented C APIs diff --git a/.github/workflows/reusable-check-html-ids.yml b/.github/workflows/reusable-check-html-ids.yml new file mode 100644 index 00000000000000..03ed714ca585fe --- /dev/null +++ b/.github/workflows/reusable-check-html-ids.yml @@ -0,0 +1,66 @@ +name: Reusable check HTML IDs + +on: + workflow_call: + +permissions: {} + +env: + FORCE_COLOR: 1 + +jobs: + check-html-ids: + name: 'Check for removed HTML IDs' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: 'Check out PR head' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + ref: ${{ github.event.pull_request.head.sha }} + - name: 'Find merge base' + id: merge-base + run: | + BASE="${{ github.event.pull_request.base.sha }}" + HEAD="${{ github.event.pull_request.head.sha }}" + git fetch --depth=$((${{ github.event.pull_request.commits }} + 10)) --no-tags origin "$BASE" "$HEAD" + + if ! MERGE_BASE=$(git merge-base "$BASE" "$HEAD" 2>/dev/null); then + git fetch --deepen=1 --no-tags origin "$BASE" "$HEAD" + + OLDEST=$(git rev-list --reflog --max-parents=0 --reverse "${BASE}^" "${HEAD}^" | head -1) + TIMESTAMP=$(git show --format=%at --no-patch "$OLDEST") + + git fetch --shallow-since="$TIMESTAMP" --no-tags origin "$BASE" "$HEAD" + + MERGE_BASE=$(git merge-base "$BASE" "$HEAD") + fi + echo "sha=$MERGE_BASE" >> "$GITHUB_OUTPUT" + - name: 'Create worktree at merge base' + env: + MERGE_BASE: ${{ steps.merge-base.outputs.sha }} + run: git worktree add /tmp/merge-base "$MERGE_BASE" --detach + - name: 'Set up Python' + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3' + cache: 'pip' + cache-dependency-path: 'Doc/requirements.txt' + - name: 'Install build dependencies' + run: make -C /tmp/merge-base/Doc/ venv + - name: 'Build HTML documentation' + run: make -C /tmp/merge-base/Doc/ SPHINXOPTS="--quiet" html + - name: 'Collect HTML IDs' + run: python Doc/tools/check-html-ids.py collect /tmp/merge-base/Doc/build/html -o /tmp/html-ids-base.json.gz + - name: 'Download PR head HTML IDs' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: html-ids-head.json.gz + path: /tmp + - name: 'Check for removed HTML IDs' + run: | + # shellcheck disable=SC2046 + python Doc/tools/check-html-ids.py -v check \ + /tmp/html-ids-base.json.gz /tmp/html-ids-head.json.gz \ + $([ -f Doc/tools/removed-ids.txt ] && echo "--exclude-file Doc/tools/removed-ids.txt") diff --git a/.github/workflows/reusable-cifuzz.yml b/.github/workflows/reusable-cifuzz.yml index 6cd9c26037f527..093b2c859eff7b 100644 --- a/.github/workflows/reusable-cifuzz.yml +++ b/.github/workflows/reusable-cifuzz.yml @@ -13,6 +13,8 @@ on: required: true type: string +permissions: {} + jobs: cifuzz: name: ${{ inputs.oss-fuzz-project-name }} (${{ inputs.sanitizer }}) @@ -21,12 +23,12 @@ jobs: steps: - name: Build fuzzers (${{ inputs.sanitizer }}) id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@ed23f8af80ff82b25ca67cd9b101e690b8897b3f # master with: oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }} sanitizer: ${{ inputs.sanitizer }} - name: Run fuzzers (${{ inputs.sanitizer }}) - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@ed23f8af80ff82b25ca67cd9b101e690b8897b3f # master with: fuzz-seconds: 600 oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }} @@ -34,13 +36,13 @@ jobs: sanitizer: ${{ inputs.sanitizer }} - name: Upload crash if: failure() && steps.build.outcome == 'success' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: ${{ inputs.sanitizer }}-artifacts path: ./out/artifacts - name: Upload SARIF if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v4 + uses: github/codeql-action/upload-sarif@38697555549f1db7851b81482ff19f1fa5c4fedc # v4.34.1 with: sarif_file: cifuzz-sarif/results.sarif checkout_path: cifuzz-sarif diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index d958d729168e23..cc9841ebf32f27 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -41,6 +41,9 @@ on: # yamllint disable-line rule:truthy run-ubuntu: description: Whether to run the Ubuntu tests value: ${{ jobs.compute-changes.outputs.run-ubuntu }} # bool + run-emscripten: + description: Whether to run the Emscripten tests + value: ${{ jobs.compute-changes.outputs.run-emscripten }} # bool run-wasi: description: Whether to run the WASI tests value: ${{ jobs.compute-changes.outputs.run-wasi }} # bool @@ -51,6 +54,8 @@ on: # yamllint disable-line rule:truthy description: Whether to run the Windows tests value: ${{ jobs.compute-changes.outputs.run-windows-tests }} # bool +permissions: {} + jobs: compute-changes: name: Create context from changed files @@ -65,19 +70,20 @@ jobs: run-macos: ${{ steps.changes.outputs.run-macos }} run-tests: ${{ steps.changes.outputs.run-tests }} run-ubuntu: ${{ steps.changes.outputs.run-ubuntu }} + run-emscripten: ${{ steps.changes.outputs.run-emscripten }} run-wasi: ${{ steps.changes.outputs.run-wasi }} run-windows-msi: ${{ steps.changes.outputs.run-windows-msi }} run-windows-tests: ${{ steps.changes.outputs.run-windows-tests }} steps: - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3" - run: >- echo '${{ github.event_name }}' - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false ref: >- diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index c1e58fd44d3790..3d534feb2ed3ea 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -4,8 +4,7 @@ on: workflow_call: workflow_dispatch: -permissions: - contents: read +permissions: {} concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -27,7 +26,7 @@ jobs: refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' steps: - name: 'Check out latest PR branch commit' - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false ref: >- @@ -52,7 +51,7 @@ jobs: git fetch origin "${refspec_base}" --shallow-since="${DATE}" \ --no-tags --prune --no-recurse-submodules - name: 'Set up Python' - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3' cache: 'pip' @@ -75,6 +74,22 @@ jobs: --fail-if-regression \ --fail-if-improved \ --fail-if-new-news-nit + - name: 'Collect HTML IDs' + if: github.event_name == 'pull_request' + run: python Doc/tools/check-html-ids.py collect Doc/build/html -o Doc/build/html-ids-head.json.gz + - name: 'Upload HTML IDs' + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: html-ids-head + path: Doc/build/html-ids-head.json.gz + archive: false + + check-html-ids: + name: 'Check for removed HTML IDs' + needs: build-doc + if: github.event_name == 'pull_request' + uses: ./.github/workflows/reusable-check-html-ids.yml # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release doctest: @@ -82,10 +97,10 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/cache@v5 + - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ~/.cache/pip key: ubuntu-doc-${{ hashFiles('Doc/requirements.txt') }} @@ -108,11 +123,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: 'Set up Python' - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3' cache: 'pip' diff --git a/.github/workflows/reusable-emscripten.yml b/.github/workflows/reusable-emscripten.yml new file mode 100644 index 00000000000000..300731deb78959 --- /dev/null +++ b/.github/workflows/reusable-emscripten.yml @@ -0,0 +1,76 @@ +name: Reusable Emscripten + +on: + workflow_call: + +permissions: {} + +env: + FORCE_COLOR: 1 + +jobs: + build-emscripten-reusable: + name: 'build and test' + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: "Read Emscripten config" + id: emscripten-config + shell: python + run: | + import hashlib + import json + import os + import tomllib + from pathlib import Path + + config = tomllib.loads(Path("Platforms/emscripten/config.toml").read_text()) + h = hashlib.sha256() + h.update(json.dumps(config["dependencies"], sort_keys=True).encode()) + h.update(Path("Platforms/emscripten/make_libffi.sh").read_bytes()) + h.update(b'1') # Update to explicitly bust cache + emsdk_cache = Path(os.environ["RUNNER_TEMP"]) / "emsdk-cache" + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"emscripten-version={config['emscripten-version']}\n") + f.write(f"node-version={config['node-version']}\n") + f.write(f"deps-hash={h.hexdigest()}\n") + with open(os.environ["GITHUB_ENV"], "a") as f: + f.write(f"EMSDK_CACHE={emsdk_cache}\n") + - name: "Install Node.js" + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: ${{ steps.emscripten-config.outputs.node-version }} + - name: "Cache Emscripten SDK" + id: emsdk-cache + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: ${{ env.EMSDK_CACHE }} + key: emsdk-${{ steps.emscripten-config.outputs.emscripten-version }}-${{ steps.emscripten-config.outputs.deps-hash }} + restore-keys: emsdk-${{ steps.emscripten-config.outputs.emscripten-version }} + - name: "Install Python" + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.x' + - name: "Runner image version" + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: "Install Emscripten" + run: python3 Platforms/emscripten install-emscripten + - name: "Configure build Python" + run: python3 Platforms/emscripten configure-build-python -- --config-cache --with-pydebug + - name: "Make build Python" + run: python3 Platforms/emscripten make-build-python + - name: "Make dependencies" + run: >- + python3 Platforms/emscripten make-dependencies + ${{ steps.emsdk-cache.outputs.cache-hit == 'true' && '--check-up-to-date' || '' }} + - name: "Configure host Python" + run: python3 Platforms/emscripten configure-host --host-runner node -- --config-cache + - name: "Make host Python" + run: python3 Platforms/emscripten make-host + - name: "Display build info" + run: python3 Platforms/emscripten run --pythoninfo + - name: "Test" + run: python3 Platforms/emscripten run --test diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 6afbf6595d93e3..a372d5715290db 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -12,6 +12,8 @@ on: required: true type: string +permissions: {} + env: FORCE_COLOR: 1 @@ -28,7 +30,7 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Runner image version diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index 79a4ded09fc9ca..33cfd578d6819a 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -12,6 +12,8 @@ on: type: boolean default: false +permissions: {} + env: FORCE_COLOR: 1 @@ -26,7 +28,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Runner image version @@ -96,7 +98,7 @@ jobs: run: find "${GITHUB_WORKSPACE}" -name 'san_log.*' | xargs head -n 1000 - name: Archive logs if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: >- ${{ inputs.sanitizer }}-logs-${{ diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 6464590dee4776..b2ab525c976330 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -23,6 +23,8 @@ on: type: string default: '' +permissions: {} + env: FORCE_COLOR: 1 @@ -36,7 +38,7 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Register gcc problem matcher @@ -56,7 +58,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v5 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ inputs.os }}-multissl-openssl-${{ env.OPENSSL_VER }} diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 8d76679a400c7f..83f9d2399ce100 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -3,6 +3,8 @@ name: Reusable WASI on: workflow_call: +permissions: {} + env: FORCE_COLOR: 1 @@ -16,12 +18,12 @@ jobs: CROSS_BUILD_PYTHON: cross-build/build CROSS_BUILD_WASI: cross-build/wasm32-wasip1 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false # No problem resolver registered as one doesn't currently exist for Clang. - name: "Install wasmtime" - uses: bytecodealliance/actions/wasmtime/setup@v1 + uses: bytecodealliance/actions/wasmtime/setup@9152e710e9f7182e4c29ad218e4f335a7b203613 # v1.1.3 with: version: ${{ env.WASMTIME_VERSION }} - name: "Read WASI SDK version" @@ -42,7 +44,7 @@ jobs: version: ${{ steps.wasi-sdk-version.outputs.version }} add-to-path: false - name: "Install Python" - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.x' - name: "Runner image version" diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml index 42c0dfd9636d30..7c724f184f3ef6 100644 --- a/.github/workflows/reusable-windows-msi.yml +++ b/.github/workflows/reusable-windows-msi.yml @@ -8,8 +8,7 @@ on: required: true type: string -permissions: - contents: read +permissions: {} env: FORCE_COLOR: 1 @@ -23,7 +22,7 @@ jobs: ARCH: ${{ inputs.arch }} IncludeFreethreaded: true steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Build CPython installer diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 2f667ace9194d7..2cfe338a6525e6 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -12,6 +12,12 @@ on: required: false type: boolean default: false + interpreter: + description: Which interpreter to build (switch-case or tail-call) + required: true + type: string + +permissions: {} env: FORCE_COLOR: 1 @@ -20,22 +26,25 @@ env: jobs: build: - name: Build and test (${{ inputs.arch }}) + name: Build and test (${{ inputs.arch }}, ${{ inputs.interpreter }}) runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2025-vs2026' }} timeout-minutes: 60 env: ARCH: ${{ inputs.arch }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Register MSVC problem matcher if: inputs.arch != 'Win32' run: echo "::add-matcher::.github/problem-matchers/msvc.json" - name: Build CPython + # msvc::musttail is not supported for debug builds, so we have to + # switch to release. run: >- .\\PCbuild\\build.bat - -e -d -v + -e -v + ${{ inputs.interpreter == 'switch-case' && '-d' || '--tail-call-interp -c Release' }} -p "${ARCH}" ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} shell: bash @@ -45,6 +54,7 @@ jobs: run: >- .\\PCbuild\\rt.bat -p "${ARCH}" - -d -q --fast-ci + -q --fast-ci + ${{ inputs.interpreter == 'switch-case' && '-d' || '' }} ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} shell: bash diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 915b1acd33f814..2c73d10350f69f 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -4,6 +4,8 @@ on: schedule: - cron: "0 */6 * * *" +permissions: {} + jobs: stale: if: github.repository_owner == 'python' @@ -14,7 +16,7 @@ jobs: steps: - name: "Check PRs" - uses: actions/stale@v10 + uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity.' diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index 32c6aa75e479f8..35c62acb28b761 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -11,8 +11,7 @@ on: paths: *paths workflow_dispatch: -permissions: - contents: read +permissions: {} concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -23,41 +22,6 @@ env: LLVM_VERSION: 21 jobs: - windows: - name: ${{ matrix.target }} - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - target: x86_64-pc-windows-msvc/msvc - architecture: x64 - runner: windows-2025-vs2026 - build_flags: "" - run_tests: true - - target: x86_64-pc-windows-msvc/msvc-free-threading - architecture: x64 - runner: windows-2025-vs2026 - build_flags: --disable-gil - run_tests: false - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - - uses: actions/setup-python@v6 - with: - python-version: '3.11' - - name: Build - shell: pwsh - run: | - ./PCbuild/build.bat --tail-call-interp ${{ matrix.build_flags }} -c Release -p ${{ matrix.architecture }} - - name: Test - if: matrix.run_tests - shell: pwsh - run: | - ./PCbuild/rt.bat -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - macos: name: ${{ matrix.target }} runs-on: ${{ matrix.runner }} @@ -69,12 +33,12 @@ jobs: - target: x86_64-apple-darwin/clang runner: macos-15-intel - target: aarch64-apple-darwin/clang - runner: macos-14 + runner: macos-15 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' - name: Install dependencies @@ -110,10 +74,10 @@ jobs: runner: ubuntu-24.04-arm configure_flags: --with-pydebug steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' - name: Build diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 135979078710cc..4ac25bc909b13f 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -13,8 +13,7 @@ on: - '.github/workflows/verify-ensurepip-wheels.yml' - 'Tools/build/verify_ensurepip_wheels.py' -permissions: - contents: read +permissions: {} concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -25,10 +24,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3' - name: Compare checksum of bundled wheels to the ones published on PyPI diff --git a/.github/workflows/verify-expat.yml b/.github/workflows/verify-expat.yml index 6b12b95cb11ff2..e193dfa4603e8a 100644 --- a/.github/workflows/verify-expat.yml +++ b/.github/workflows/verify-expat.yml @@ -11,8 +11,7 @@ on: - 'Modules/expat/**' - '.github/workflows/verify-expat.yml' -permissions: - contents: read +permissions: {} concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -23,7 +22,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Download and verify bundled libexpat files diff --git a/.github/zizmor.yml b/.github/zizmor.yml index 8b7b4de0fc8f31..7c776d5ea1f941 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -4,7 +4,3 @@ rules: dangerous-triggers: ignore: - documentation-links.yml - unpinned-uses: - config: - policies: - "*": ref-pin diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dfd18182105e11..6e612ce232fb29 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,9 +3,9 @@ repos: rev: a27a2e47c7751b639d2b5badf0ef6ff11fee893f # frozen: v0.15.4 hooks: - id: ruff-check - name: Run Ruff (lint) on Apple/ - args: [--exit-non-zero-on-fix, --config=Apple/.ruff.toml] - files: ^Apple/ + name: Run Ruff (lint) on Platforms/Apple/ + args: [--exit-non-zero-on-fix, --config=Platforms/Apple/.ruff.toml] + files: ^Platforms/Apple/ - id: ruff-check name: Run Ruff (lint) on Doc/ args: [--exit-non-zero-on-fix] @@ -39,9 +39,9 @@ repos: args: [--exit-non-zero-on-fix, --config=Tools/wasm/.ruff.toml] files: ^Tools/wasm/ - id: ruff-format - name: Run Ruff (format) on Apple/ - args: [--exit-non-zero-on-fix, --config=Apple/.ruff.toml] - files: ^Apple + name: Run Ruff (format) on Platforms/Apple/ + args: [--exit-non-zero-on-fix, --config=Platforms/Apple/.ruff.toml] + files: ^Platforms/Apple/ - id: ruff-format name: Run Ruff (format) on Doc/ args: [--exit-non-zero-on-fix] diff --git a/Android/android.py b/Android/android.py index b644be9cc64c7a..adcc7c708d95f7 100755 --- a/Android/android.py +++ b/Android/android.py @@ -34,7 +34,12 @@ TESTBED_DIR = ANDROID_DIR / "testbed" CROSS_BUILD_DIR = PYTHON_DIR / "cross-build" -HOSTS = ["aarch64-linux-android", "x86_64-linux-android"] +HOSTS = [ + "aarch64-linux-android", + "arm-linux-androideabi", + "i686-linux-android", + "x86_64-linux-android", +] APP_ID = "org.python.testbed" DECODE_ARGS = ("UTF-8", "backslashreplace") @@ -205,38 +210,48 @@ def make_build_python(context): # # If you're a member of the Python core team, and you'd like to be able to push # these tags yourself, please contact Malcolm Smith or Russell Keith-Magee. -def unpack_deps(host, prefix_dir): +def unpack_deps(host, prefix_dir, cache_dir): os.chdir(prefix_dir) deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.5.5-0", - "sqlite-3.50.4-0", "xz-5.4.6-1", "zstd-1.5.7-1"]: + "sqlite-3.50.4-0", "xz-5.4.6-1", "zstd-1.5.7-2"]: filename = f"{name_ver}-{host}.tar.gz" - download(f"{deps_url}/{name_ver}/{filename}") - shutil.unpack_archive(filename) - os.remove(filename) + out_path = download(f"{deps_url}/{name_ver}/{filename}", cache_dir) + shutil.unpack_archive(out_path) -def download(url, target_dir="."): - out_path = f"{target_dir}/{basename(url)}" - run(["curl", "-Lf", "--retry", "5", "--retry-all-errors", "-o", out_path, url]) +def download(url, cache_dir): + out_path = cache_dir / basename(url) + cache_dir.mkdir(parents=True, exist_ok=True) + if not out_path.is_file(): + run(["curl", "-Lf", "--retry", "5", "--retry-all-errors", "-o", out_path, url]) + else: + print(f"Using cached version of {basename(url)}") return out_path -def configure_host_python(context): +def configure_host_python(context, host=None): + if host is None: + host = context.host if context.clean: - clean(context.host) + clean(host) - host_dir = subdir(context.host, create=True) + host_dir = subdir(host, create=True) prefix_dir = host_dir / "prefix" if not prefix_dir.exists(): prefix_dir.mkdir() - unpack_deps(context.host, prefix_dir) + cache_dir = ( + Path(context.cache_dir).resolve() + if context.cache_dir + else CROSS_BUILD_DIR / "downloads" + ) + unpack_deps(host, prefix_dir, cache_dir) os.chdir(host_dir) command = [ # Basic cross-compiling configuration relpath(PYTHON_DIR / "configure"), - f"--host={context.host}", + f"--host={host}", f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", f"--with-build-python={build_python_path()}", "--without-ensurepip", @@ -252,14 +267,16 @@ def configure_host_python(context): if context.args: command.extend(context.args) - run(command, host=context.host) + run(command, host=host) -def make_host_python(context): +def make_host_python(context, host=None): + if host is None: + host = context.host # The CFLAGS and LDFLAGS set in android-env include the prefix dir, so # delete any previous Python installation to prevent it being used during # the build. - host_dir = subdir(context.host) + host_dir = subdir(host) prefix_dir = host_dir / "prefix" for pattern in ("include/python*", "lib/libpython*", "lib/python*"): delete_glob(f"{prefix_dir}/{pattern}") @@ -278,20 +295,28 @@ def make_host_python(context): ) -def build_all(context): - steps = [configure_build_python, make_build_python, configure_host_python, - make_host_python] - for step in steps: - step(context) +def build_targets(context): + if context.target in {"all", "build"}: + configure_build_python(context) + make_build_python(context) + + for host in HOSTS: + if context.target in {"all", "hosts", host}: + configure_host_python(context, host) + make_host_python(context, host) def clean(host): delete_glob(CROSS_BUILD_DIR / host) -def clean_all(context): - for host in HOSTS + ["build"]: - clean(host) +def clean_targets(context): + if context.target in {"all", "build"}: + clean("build") + + for host in HOSTS: + if context.target in {"all", "hosts", host}: + clean(host) def setup_ci(): @@ -628,7 +653,8 @@ async def gradle_task(context): # Randomization is disabled because order-dependent failures are # much less likely to pass on a rerun in single-process mode. "-m", "test", - f"--{context.ci_mode}-ci", "--single-process", "--no-randomize" + f"--{context.ci_mode}-ci", "--single-process", "--no-randomize", + "--pythoninfo", ] if not any(arg in context.args for arg in ["-c", "-m"]): @@ -853,18 +879,23 @@ def add_parser(*args, **kwargs): # Subcommands build = add_parser( - "build", help="Run configure-build, make-build, configure-host and " - "make-host") + "build", + help="Run configure and make for the selected target" + ) configure_build = add_parser( "configure-build", help="Run `configure` for the build Python") - add_parser( + make_build = add_parser( "make-build", help="Run `make` for the build Python") configure_host = add_parser( "configure-host", help="Run `configure` for Android") make_host = add_parser( "make-host", help="Run `make` for Android") - add_parser("clean", help="Delete all build directories") + clean = add_parser( + "clean", + help="Delete build directories for the selected target" + ) + add_parser("build-testbed", help="Build the testbed app") test = add_parser("test", help="Run the testbed app") package = add_parser("package", help="Make a release package") @@ -872,12 +903,61 @@ def add_parser(*args, **kwargs): env = add_parser("env", help="Print environment variables") # Common arguments + # --cross-build-dir argument + for cmd in [ + clean, + configure_build, + make_build, + configure_host, + make_host, + build, + package, + test, + ci, + ]: + cmd.add_argument( + "--cross-build-dir", + action="store", + default=os.environ.get("CROSS_BUILD_DIR"), + dest="cross_build_dir", + type=Path, + help=( + "Path to the cross-build directory " + f"(default: {CROSS_BUILD_DIR}). Can also be set " + "with the CROSS_BUILD_DIR environment variable." + ), + ) + + # --cache-dir option + for cmd in [configure_host, build, ci]: + cmd.add_argument( + "--cache-dir", + default=os.environ.get("CACHE_DIR"), + help="The directory to store cached downloads.", + ) + + # --clean option for subcommand in [build, configure_build, configure_host, ci]: subcommand.add_argument( "--clean", action="store_true", default=False, dest="clean", help="Delete the relevant build directories first") - host_commands = [build, configure_host, make_host, package, ci] + # Allow "all", "build" and "hosts" targets for some commands + for subcommand in [clean, build]: + subcommand.add_argument( + "target", + nargs="?", + default="all", + choices=["all", "build", "hosts"] + HOSTS, + help=( + "The host triplet (e.g., aarch64-linux-android), " + "or 'build' for just the build platform, or 'hosts' for all " + "host platforms, or 'all' for the build platform and all " + "hosts. Defaults to 'all'" + ), + ) + + host_commands = [configure_host, make_host, package, ci] if in_source_tree: host_commands.append(env) for subcommand in host_commands: @@ -939,13 +1019,19 @@ def main(): stream.reconfigure(line_buffering=True) context = parse_args() + + # Set the CROSS_BUILD_DIR if an argument was provided + if context.cross_build_dir: + global CROSS_BUILD_DIR + CROSS_BUILD_DIR = context.cross_build_dir.resolve() + dispatch = { "configure-build": configure_build_python, "make-build": make_build_python, "configure-host": configure_host_python, "make-host": make_host_python, - "build": build_all, - "clean": clean_all, + "build": build_targets, + "clean": clean_targets, "build-testbed": build_testbed, "test": run_testbed, "package": package, diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts index 53cdc591fa35fd..7529fdb8f7852f 100644 --- a/Android/testbed/app/build.gradle.kts +++ b/Android/testbed/app/build.gradle.kts @@ -15,6 +15,8 @@ val inSourceTree = ( val KNOWN_ABIS = mapOf( "aarch64-linux-android" to "arm64-v8a", + "arm-linux-androideabi" to "armeabi-v7a", + "i686-linux-android" to "x86", "x86_64-linux-android" to "x86_64", ) diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index e2b22ec3c794ae..2b36da997d4295 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -44,6 +44,10 @@ Direct API functions On failure, return ``NULL`` with an exception set. + .. note:: + If the object implements the buffer protocol, then the buffer + must not be mutated while the bytearray object is being created. + .. c:function:: PyObject* PyByteArray_FromStringAndSize(const char *string, Py_ssize_t len) @@ -58,6 +62,10 @@ Direct API functions On failure, return ``NULL`` with an exception set. + .. note:: + If the object implements the buffer protocol, then the buffer + must not be mutated while the bytearray object is being created. + .. c:function:: Py_ssize_t PyByteArray_Size(PyObject *bytearray) @@ -70,6 +78,9 @@ Direct API functions ``NULL`` pointer. The returned array always has an extra null byte appended. + .. note:: + It is not thread-safe to mutate the bytearray object while using the returned char array. + .. c:function:: int PyByteArray_Resize(PyObject *bytearray, Py_ssize_t len) @@ -89,6 +100,9 @@ These macros trade safety for speed and they don't check pointers. Similar to :c:func:`PyByteArray_AsString`, but without error checking. + .. note:: + It is not thread-safe to mutate the bytearray object while using the returned char array. + .. c:function:: Py_ssize_t PyByteArray_GET_SIZE(PyObject *bytearray) diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index b3cd26a8504715..d1fde1baf71a45 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -127,6 +127,10 @@ called with a non-bytes parameter. Return the bytes representation of object *o* that implements the buffer protocol. + .. note:: + If the object implements the buffer protocol, then the buffer + must not be mutated while the bytes object is being created. + .. c:function:: Py_ssize_t PyBytes_Size(PyObject *o) @@ -185,6 +189,9 @@ called with a non-bytes parameter. created, the old reference to *bytes* will still be discarded and the value of *\*bytes* will be set to ``NULL``; the appropriate exception will be set. + .. note:: + If *newpart* implements the buffer protocol, then the buffer + must not be mutated while the new bytes object is being created. .. c:function:: void PyBytes_ConcatAndDel(PyObject **bytes, PyObject *newpart) @@ -192,6 +199,10 @@ called with a non-bytes parameter. appended to *bytes*. This version releases the :term:`strong reference` to *newpart* (i.e. decrements its reference count). + .. note:: + If *newpart* implements the buffer protocol, then the buffer + must not be mutated while the new bytes object is being created. + .. c:function:: PyObject* PyBytes_Join(PyObject *sep, PyObject *iterable) @@ -210,6 +221,9 @@ called with a non-bytes parameter. .. versionadded:: 3.14 + .. note:: + If *iterable* objects implement the buffer protocol, then the buffers + must not be mutated while the new bytes object is being created. .. c:function:: int _PyBytes_Resize(PyObject **bytes, Py_ssize_t newsize) diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index e23288c6a58590..b913e24b3c7cc8 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -8,13 +8,31 @@ Descriptor Objects "Descriptors" are objects that describe some attribute of an object. They are found in the dictionary of type objects. -.. XXX document these! - .. c:function:: PyObject* PyDescr_NewGetSet(PyTypeObject *type, struct PyGetSetDef *getset) + Create a new get-set descriptor for extension type *type* from the + :c:type:`PyGetSetDef` structure *getset*. + + Get-set descriptors expose attributes implemented by C getter and setter + functions rather than stored directly in the instance. This is the same kind + of descriptor created for entries in :c:member:`~PyTypeObject.tp_getset`, and + it appears in Python as a :class:`types.GetSetDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. + +.. c:function:: PyObject* PyDescr_NewMember(PyTypeObject *type, struct PyMemberDef *member) -.. c:function:: PyObject* PyDescr_NewMember(PyTypeObject *type, struct PyMemberDef *meth) + Create a new member descriptor for extension type *type* from the + :c:type:`PyMemberDef` structure *member*. + Member descriptors expose fields in the type's C struct as Python + attributes. This is the same kind of descriptor created for entries in + :c:member:`~PyTypeObject.tp_members`, and it appears in Python as a + :class:`types.MemberDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. .. c:var:: PyTypeObject PyMemberDescr_Type @@ -30,22 +48,53 @@ found in the dictionary of type objects. The type object for get/set descriptor objects created from :c:type:`PyGetSetDef` structures. These descriptors implement attributes whose value is computed by C getter and setter functions, and are used - for many built-in type attributes. + for many built-in type attributes. They correspond to + :class:`types.GetSetDescriptorType` objects in Python. .. c:function:: PyObject* PyDescr_NewMethod(PyTypeObject *type, struct PyMethodDef *meth) + Create a new method descriptor for extension type *type* from the + :c:type:`PyMethodDef` structure *meth*. + + Method descriptors expose C functions as methods on a type. This is the same + kind of descriptor created for entries in + :c:member:`~PyTypeObject.tp_methods`, and it appears in Python as a + :class:`types.MethodDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. .. c:var:: PyTypeObject PyMethodDescr_Type The type object for method descriptor objects created from :c:type:`PyMethodDef` structures. These descriptors expose C functions as - methods on a type, and correspond to :class:`types.MemberDescriptorType` + methods on a type, and correspond to :class:`types.MethodDescriptorType` objects in Python. -.. c:function:: PyObject* PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *wrapper, void *wrapped) +.. c:struct:: wrapperbase + + Describes a slot wrapper used by :c:func:`PyDescr_NewWrapper`. + Each ``wrapperbase`` record stores the Python-visible name and metadata for a + special method implemented by a type slot, together with the wrapper + function used to adapt that slot to Python's calling convention. + +.. c:function:: PyObject* PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *base, void *wrapped) + + Create a new wrapper descriptor for extension type *type* from the + :c:struct:`wrapperbase` structure *base* and the wrapped slot function + pointer + *wrapped*. + + Wrapper descriptors expose special methods implemented by type slots. This + is the same kind of descriptor that CPython creates for slot-based special + methods such as ``__repr__`` or ``__add__``, and it appears in Python as a + :class:`types.WrapperDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. .. c:var:: PyTypeObject PyWrapperDescr_Type @@ -58,6 +107,16 @@ found in the dictionary of type objects. .. c:function:: PyObject* PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method) + Create a new class method descriptor for extension type *type* from the + :c:type:`PyMethodDef` structure *method*. + + Class method descriptors expose C methods that receive the class rather than + an instance when accessed. This is the same kind of descriptor created for + ``METH_CLASS`` entries in :c:member:`~PyTypeObject.tp_methods`, and it + appears in Python as a :class:`types.ClassMethodDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. .. c:function:: int PyDescr_IsData(PyObject *descr) @@ -66,8 +125,18 @@ found in the dictionary of type objects. no error checking. -.. c:function:: PyObject* PyWrapper_New(PyObject *, PyObject *) +.. c:function:: PyObject* PyWrapper_New(PyObject *d, PyObject *self) + + Create a new bound wrapper object from the wrapper descriptor *d* and the + instance *self*. + + This is the bound form of a wrapper descriptor created by + :c:func:`PyDescr_NewWrapper`. CPython creates these objects when a slot + wrapper is accessed through an instance, and they appear in Python as + :class:`types.MethodWrapperType` objects. + On success, return a :term:`strong reference` to the wrapper object. Return + ``NULL`` with an exception set on failure. .. c:macro:: PyDescr_COMMON @@ -104,9 +173,9 @@ Built-in descriptors .. c:var:: PyTypeObject PyClassMethodDescr_Type The type object for C-level class method descriptor objects. - This is the type of the descriptors created for :func:`classmethod` defined in - C extension types, and is the same object as :class:`classmethod` - in Python. + This is the type of the descriptors created for :func:`classmethod` defined + in C extension types, and corresponds to + :class:`types.ClassMethodDescriptorType` objects in Python. .. c:function:: PyObject *PyClassMethod_New(PyObject *callable) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 8ecd7c62517104..576b58e6fbcb19 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -1346,3 +1346,67 @@ Tracebacks This function returns ``0`` on success, and returns ``-1`` with an exception set on failure. + +.. c:function:: const char* PyUnstable_DumpTraceback(int fd, PyThreadState *tstate) + + Write a trace of the Python stack in *tstate* into the file *fd*. The format + looks like:: + + Traceback (most recent call first): + File "xxx", line xxx in + File "xxx", line xxx in + ... + File "xxx", line xxx in + + This function is meant to debug situations such as segfaults, fatal errors, + and similar. The file and function names it outputs are encoded to ASCII with + backslashreplace and truncated to 500 characters. It writes only the first + 100 frames; further frames are truncated with the line ``...``. + + This function will return ``NULL`` on success, or an error message on error. + + This function is intended for use in crash scenarios such as signal handlers + for SIGSEGV, where the interpreter may be in an inconsistent state. Given + that it reads interpreter data structures that may be partially modified, the + function might produce incomplete output or it may even crash itself. + + The caller does not need to hold an :term:`attached thread state`, nor does + *tstate* need to be attached. + + .. versionadded:: next + +.. c:function:: const char* PyUnstable_DumpTracebackThreads(int fd, PyInterpreterState *interp, PyThreadState *current_tstate) + + Write the traces of all Python threads in *interp* into the file *fd*. + + If *interp* is ``NULL`` then this function will try to identify the current + interpreter using thread-specific storage. If it cannot, it will return an + error. + + If *current_tstate* is not ``NULL`` then it will be used to identify what the + current thread is in the written output. If it is ``NULL`` then this function + will identify the current thread using thread-specific storage. It is not an + error if the function is unable to get the current Python thread state. + + This function will return ``NULL`` on success, or an error message on error. + It will also write this error message to *fd*. + + This function is meant to debug debug situations such as segfaults, fatal + errors, and similar. It calls :c:func:`PyUnstable_DumpTraceback` for each + thread. It only writes the tracebacks of the first 100 threads, further + output is truncated with the line ``...``. + + This function is intended for use in crash scenarios such as signal handlers + for SIGSEGV, where the interpreter may be in an inconsistent state. Given + that it reads interpreter data structures that may be partially modified, the + function might produce incomplete output or it may even crash itself. + + The caller does not need to hold an :term:`attached thread state`, nor does + *current_tstate* need to be attached. + + .. warning:: + On the :term:`free-threaded build`, this function is not thread-safe. If + another thread deletes its :term:`thread state` while this function is being + called, the process will likely crash. + + .. versionadded:: next diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index c3a80234f86116..e42c1dbf420069 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -526,14 +526,26 @@ to the C language. Outdated macros --------------- -The following macros have been used to features that have been standardized -in C11. +The following :term:`soft deprecated` macros have been used to features that +have been standardized in C11 (or previous standards). .. c:macro:: Py_ALIGNED(num) - Specify alignment to *num* bytes on compilers that support it. + On some GCC-like compilers, specify alignment to *num* bytes. + This does nothing on other compilers. - Consider using the C11 standard ``_Alignas`` specifier over this macro. + Use the standard ``alignas`` specifier rather than this macro. + + .. deprecated:: next + The macro is :term:`soft deprecated`. + +.. c:macro:: PY_FORMAT_SIZE_T + + The :c:func:`printf` formatting modifier for :c:type:`size_t`. + Use ``"z"`` directly instead. + + .. deprecated:: next + The macro is :term:`soft deprecated`. .. c:macro:: Py_LL(number) Py_ULL(number) @@ -546,6 +558,38 @@ in C11. Consider using the C99 standard suffixes ``LL`` and ``LLU`` directly. + .. deprecated:: next + The macro is :term:`soft deprecated`. + +.. c:macro:: PY_LONG_LONG + PY_INT32_T + PY_UINT32_T + PY_INT64_T + PY_UINT64_T + + Aliases for the types :c:type:`!long long`, :c:type:`!int32_t`, + :c:type:`!uint32_t`. :c:type:`!int64_t` and :c:type:`!uint64_t`, + respectively. + Historically, these types needed compiler-specific extensions. + + .. deprecated:: next + These macros are :term:`soft deprecated`. + +.. c:macro:: PY_LLONG_MIN + PY_LLONG_MAX + PY_ULLONG_MAX + PY_SIZE_MAX + + Aliases for the values :c:macro:`!LLONG_MIN`, :c:macro:`!LLONG_MAX`, + :c:macro:`!ULLONG_MAX`, and :c:macro:`!SIZE_MAX`, respectively. + Use these standard names instead. + + The required header, ````, + :ref:`is included ` in ``Python.h``. + + .. deprecated:: next + These macros are :term:`soft deprecated`. + .. c:macro:: Py_MEMCPY(dest, src, n) This is a :term:`soft deprecated` alias to :c:func:`!memcpy`. @@ -554,6 +598,25 @@ in C11. .. deprecated:: 3.14 The macro is :term:`soft deprecated`. +.. c:macro:: Py_UNICODE_SIZE + + Size of the :c:type:`!wchar_t` type. + Use ``sizeof(wchar_t)`` or ``WCHAR_WIDTH/8`` instead. + + The required header for the latter, ````, + :ref:`is included ` in ``Python.h``. + + .. deprecated:: next + The macro is :term:`soft deprecated`. + +.. c:macro:: Py_UNICODE_WIDE + + Defined if ``wchar_t`` can hold a Unicode character (UCS-4). + Use ``sizeof(wchar_t) >= 4`` instead + + .. deprecated:: next + The macro is :term:`soft deprecated`. + .. c:macro:: Py_VA_COPY This is a :term:`soft deprecated` alias to the C99-standard ``va_copy`` @@ -564,6 +627,9 @@ in C11. .. versionchanged:: 3.6 This is now an alias to ``va_copy``. + .. deprecated:: next + The macro is :term:`soft deprecated`. + .. _api-objects: diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 39293b0fa228df..8b967c285ac865 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -230,6 +230,9 @@ Feature slots When creating a module, Python checks the value of this slot using :c:func:`PyABIInfo_Check`. + This slot is required, except for modules created from + :c:struct:`PyModuleDef`. + .. versionadded:: 3.15 .. c:macro:: Py_mod_multiple_interpreters @@ -620,9 +623,9 @@ rather than from an extension's :ref:`export hook `. and the :py:class:`~importlib.machinery.ModuleSpec` *spec*. The *slots* argument must point to an array of :c:type:`PyModuleDef_Slot` - structures, terminated by an entry slot with slot ID of 0 + structures, terminated by an entry with slot ID of 0 (typically written as ``{0}`` or ``{0, NULL}`` in C). - The *slots* argument may not be ``NULL``. + The array must include a :c:data:`Py_mod_abi` entry. The *spec* argument may be any ``ModuleSpec``-like object, as described in :c:macro:`Py_mod_create` documentation. diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index f5e6b7ad157e99..f8b41f6d87f975 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -88,6 +88,16 @@ Contents of the Limited API are :ref:`listed below `. You can also define ``Py_LIMITED_API`` to ``3``. This works the same as ``0x03020000`` (Python 3.2, the version that introduced Limited API). +.. c:macro:: Py_TARGET_ABI3T + + Define this macro before including ``Python.h`` to opt in to only use + the Limited API for :term:`free-threaded builds `, + and to select the Limited API version. + + .. seealso:: :pep:`803` + + .. versionadded:: next + .. _stable-abi: diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 3a9b34b592268a..679823948e73b5 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1855,8 +1855,6 @@ object. On success, return ``0``. On error, set an exception, leave the writer unchanged, and return ``-1``. - .. versionadded:: 3.14 - .. c:function:: int PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *writer, const wchar_t *str, Py_ssize_t size) Write the wide string *str* into *writer*. @@ -1892,6 +1890,10 @@ object. On success, return ``0``. On error, set an exception, leave the writer unchanged, and return ``-1``. + .. versionchanged:: 3.14.4 + + Added support for ``NULL``. + .. c:function:: int PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, Py_ssize_t start, Py_ssize_t end) Write the substring ``str[start:end]`` into *writer*. diff --git a/Doc/conf.py b/Doc/conf.py index 4ac6f6192a0806..e2dff74538a342 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -73,6 +73,7 @@ # General substitutions. project = 'Python' copyright = "2001 Python Software Foundation" +_doc_authors = 'Python documentation authors' # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. @@ -177,6 +178,7 @@ ('c:type', '__int64'), ('c:type', 'unsigned __int64'), ('c:type', 'double'), + ('c:type', '_Float16'), # Standard C structures ('c:struct', 'in6_addr'), ('c:struct', 'in_addr'), @@ -360,69 +362,74 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). -_stdauthor = 'The Python development team' latex_documents = [ - ('c-api/index', 'c-api.tex', 'The Python/C API', _stdauthor, 'manual'), + ('c-api/index', 'c-api.tex', 'The Python/C API', _doc_authors, 'manual'), ( 'extending/index', 'extending.tex', 'Extending and Embedding Python', - _stdauthor, + _doc_authors, 'manual', ), ( 'installing/index', 'installing.tex', 'Installing Python Modules', - _stdauthor, + _doc_authors, 'manual', ), ( 'library/index', 'library.tex', 'The Python Library Reference', - _stdauthor, + _doc_authors, 'manual', ), ( 'reference/index', 'reference.tex', 'The Python Language Reference', - _stdauthor, + _doc_authors, 'manual', ), ( 'tutorial/index', 'tutorial.tex', 'Python Tutorial', - _stdauthor, + _doc_authors, 'manual', ), ( 'using/index', 'using.tex', 'Python Setup and Usage', - _stdauthor, + _doc_authors, 'manual', ), ( 'faq/index', 'faq.tex', 'Python Frequently Asked Questions', - _stdauthor, + _doc_authors, 'manual', ), ( 'whatsnew/' + version, 'whatsnew.tex', 'What\'s New in Python', - 'A. M. Kuchling', + _doc_authors, 'howto', ), ] # Collect all HOWTOs individually latex_documents.extend( - ('howto/' + fn[:-4], 'howto-' + fn[:-4] + '.tex', '', _stdauthor, 'howto') + ( + 'howto/' + fn[:-4], + 'howto-' + fn[:-4] + '.tex', + '', + _doc_authors, + 'howto', + ) for fn in os.listdir('howto') if fn.endswith('.rst') and fn != 'index.rst' ) @@ -433,7 +440,7 @@ # Options for Epub output # ----------------------- -epub_author = 'Python Documentation Authors' +epub_author = _doc_authors epub_publisher = 'Python Software Foundation' epub_exclude_files = ( 'index.xhtml', @@ -571,6 +578,17 @@ stable_abi_file = 'data/stable_abi.dat' threadsafety_file = 'data/threadsafety.dat' +# Options for notfound.extension +# ------------------------------- + +if not os.getenv("READTHEDOCS"): + if language_code: + notfound_urls_prefix = ( + f'/{language_code.replace("_", "-").lower()}/{version}/' + ) + else: + notfound_urls_prefix = f'/{version}/' + # Options for sphinxext-opengraph # ------------------------------- diff --git a/Doc/data/threadsafety.dat b/Doc/data/threadsafety.dat index 103e8ef3e97ed1..82edd1167ef128 100644 --- a/Doc/data/threadsafety.dat +++ b/Doc/data/threadsafety.dat @@ -66,10 +66,90 @@ PyList_Reverse:shared: # is a list PyList_SetSlice:shared: -# Sort - per-object lock held; comparison callbacks may execute -# arbitrary Python code +# Sort - per-object lock held; the list is emptied before sorting +# so other threads may observe an empty list, but they won't see the +# intermediate states of the sort PyList_Sort:shared: # Extend - lock target list; also lock source when it is a # list, set, or dict PyList_Extend:shared: + +# Creation - pure allocation, no shared state +PyBytes_FromString:atomic: +PyBytes_FromStringAndSize:atomic: +PyBytes_DecodeEscape:atomic: + +# Creation from formatting C primitives - pure allocation, no shared state +PyBytes_FromFormat:atomic: +PyBytes_FromFormatV:atomic: + +# Creation from object - uses buffer protocol so may call arbitrary code; +# safe as long as the buffer is not mutated by another thread during the operation +PyBytes_FromObject:shared: + +# Size - uses atomic load on free-threaded builds +PyBytes_Size:atomic: +PyBytes_GET_SIZE:atomic: + +# Raw data - no locking; mutating it is unsafe if the bytes object is shared between threads +PyBytes_AsString:compatible: +PyBytes_AS_STRING:compatible: +PyBytes_AsStringAndSize:compatible: + +# Concatenation - uses buffer protocol; safe as long as buffer is not mutated by another thread during the operation +PyBytes_Concat:shared: +PyBytes_ConcatAndDel:shared: +PyBytes_Join:shared: + +# Resizing - safe if the object is unique +_PyBytes_Resize:distinct: + +# Repr - atomic as bytes are immutable +PyBytes_Repr:atomic: + +# Creation from object - may call arbitrary code +PyByteArray_FromObject:shared: + +# Creation - pure allocation, no shared state +PyByteArray_FromStringAndSize:atomic: + +# Concatenation - uses buffer protocol; safe as long as buffer is not mutated by another thread during the operation +PyByteArray_Concat:shared: + +# Size - uses atomic load on free-threaded builds +PyByteArray_Size:atomic: +PyByteArray_GET_SIZE:atomic: + +# Raw data - no locking; mutating it is unsafe if the bytearray object is shared between threads +PyByteArray_AsString:compatible: +PyByteArray_AS_STRING:compatible: + +# Capsule objects (Doc/c-api/capsule.rst) + +# Type check - read ob_type pointer, always safe +PyCapsule_CheckExact:atomic: + +# Creation - pure allocation, no shared state +PyCapsule_New:atomic: + +# Validation - reads pointer and name fields; safe on distinct objects +PyCapsule_IsValid:distinct: + +# Getters - read struct fields; safe on distinct objects but +# concurrent access to the same capsule requires external synchronization +PyCapsule_GetPointer:distinct: +PyCapsule_GetName:distinct: +PyCapsule_GetDestructor:distinct: +PyCapsule_GetContext:distinct: + +# Setters - write struct fields; safe on distinct objects but +# concurrent access to the same capsule requires external synchronization +PyCapsule_SetPointer:distinct: +PyCapsule_SetName:distinct: +PyCapsule_SetDestructor:distinct: +PyCapsule_SetContext:distinct: + +# Import - looks up a capsule from a module attribute and +# calls PyCapsule_GetPointer; may call arbitrary code +PyCapsule_Import:compatible: diff --git a/Doc/deprecations/c-api-pending-removal-in-3.15.rst b/Doc/deprecations/c-api-pending-removal-in-3.15.rst index 9927b876760d34..789ec83d2d957a 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.15.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.15.rst @@ -7,8 +7,6 @@ Pending removal in Python 3.15 Use :c:func:`PyWeakref_GetRef` instead. The `pythoncapi-compat project `__ can be used to get :c:func:`PyWeakref_GetRef` on Python 3.12 and older. -* :c:type:`Py_UNICODE` type and the :c:macro:`!Py_UNICODE_WIDE` macro: - Use :c:type:`wchar_t` instead. * :c:func:`!PyUnicode_AsDecodedObject`: Use :c:func:`PyCodec_Decode` instead. * :c:func:`!PyUnicode_AsDecodedUnicode`: diff --git a/Doc/extending/first-extension-module.rst b/Doc/extending/first-extension-module.rst index f1ba0a3ceb7dba..cd755a98f7f5f4 100644 --- a/Doc/extending/first-extension-module.rst +++ b/Doc/extending/first-extension-module.rst @@ -265,12 +265,19 @@ Define this array just before your export hook: .. code-block:: c + PyABIInfo_VAR(abi_info); + static PyModuleDef_Slot spam_slots[] = { + {Py_mod_abi, &abi_info}, {Py_mod_name, "spam"}, {Py_mod_doc, "A wonderful module with an example function"}, {0, NULL} }; +The ``PyABIInfo_VAR(abi_info);`` macro and the :c:data:`Py_mod_abi` slot +are a bit of boilerplate that helps prevent extensions compiled for +a different version of Python from crashing the interpreter. + For both :c:data:`Py_mod_name` and :c:data:`Py_mod_doc`, the values are C strings -- that is, NUL-terminated, UTF-8 encoded byte arrays. diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 8bd2bc99d74b83..ff34bb5d71c22b 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1074,7 +1074,7 @@ Performance My program is too slow. How do I speed it up? --------------------------------------------- -That's a tough one, in general. First, here is list of things to +That's a tough one, in general. First, here is a list of things to remember before diving further: * Performance characteristics vary across Python implementations. This FAQ diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 93850b57af2c65..5260c2ca4add47 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -105,8 +105,8 @@ The complete :class:`!Weekday` enum now looks like this:: Now we can find out what today is! Observe:: - >>> from datetime import date - >>> Weekday.from_date(date.today()) # doctest: +SKIP + >>> import datetime as dt + >>> Weekday.from_date(dt.date.today()) # doctest: +SKIP Of course, if you're reading this on some other day, you'll see that day instead. @@ -1480,8 +1480,8 @@ TimePeriod An example to show the :attr:`~Enum._ignore_` attribute in use:: - >>> from datetime import timedelta - >>> class Period(timedelta, Enum): + >>> import datetime as dt + >>> class Period(dt.timedelta, Enum): ... "different lengths of time" ... _ignore_ = 'Period i' ... Period = vars() diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index b87ac93296b915..21df6ba858d8c2 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -1549,10 +1549,10 @@ to this (remembering to first import :mod:`concurrent.futures`):: for i in range(10): executor.submit(worker_process, queue, worker_configurer) -Deploying Web applications using Gunicorn and uWSGI +Deploying web applications using Gunicorn and uWSGI ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -When deploying Web applications using `Gunicorn `_ or `uWSGI +When deploying web applications using `Gunicorn `_ or `uWSGI `_ (or similar), multiple worker processes are created to handle client requests. In such environments, avoid creating file-based handlers directly in your web application. Instead, use a @@ -3616,7 +3616,6 @@ detailed information. .. code-block:: python3 - import datetime import logging import random import sys @@ -3851,7 +3850,7 @@ Logging to syslog with RFC5424 support Although :rfc:`5424` dates from 2009, most syslog servers are configured by default to use the older :rfc:`3164`, which hails from 2001. When ``logging`` was added to Python in 2003, it supported the earlier (and only existing) protocol at the time. Since -RFC5424 came out, as there has not been widespread deployment of it in syslog +RFC 5424 came out, as there has not been widespread deployment of it in syslog servers, the :class:`~logging.handlers.SysLogHandler` functionality has not been updated. @@ -3859,7 +3858,7 @@ RFC 5424 contains some useful features such as support for structured data, and need to be able to log to a syslog server with support for it, you can do so with a subclassed handler which looks something like this:: - import datetime + import datetime as dt import logging.handlers import re import socket @@ -3877,7 +3876,7 @@ subclassed handler which looks something like this:: def format(self, record): version = 1 - asctime = datetime.datetime.fromtimestamp(record.created).isoformat() + asctime = dt.datetime.fromtimestamp(record.created).isoformat() m = self.tz_offset.match(time.strftime('%z')) has_offset = False if m and time.timezone: diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst index 7486a378dbb06f..84ec535ca98e97 100644 --- a/Doc/howto/regex.rst +++ b/Doc/howto/regex.rst @@ -1,7 +1,7 @@ .. _regex-howto: **************************** - Regular Expression HOWTO + Regular expression HOWTO **************************** :Author: A.M. Kuchling @@ -47,7 +47,7 @@ Python code to do the processing; while Python code will be slower than an elaborate regular expression, it will also probably be more understandable. -Simple Patterns +Simple patterns =============== We'll start by learning about the simplest possible regular expressions. Since @@ -59,7 +59,7 @@ expressions (deterministic and non-deterministic finite automata), you can refer to almost any textbook on writing compilers. -Matching Characters +Matching characters ------------------- Most letters and characters will simply match themselves. For example, the @@ -159,7 +159,7 @@ match even a newline. ``.`` is often used where you want to match "any character". -Repeating Things +Repeating things ---------------- Being able to match varying sets of characters is the first thing regular @@ -210,7 +210,7 @@ this RE against the string ``'abcbd'``. | | | ``[bcd]*`` is only matching | | | | ``bc``. | +------+-----------+---------------------------------+ -| 6 | ``abcb`` | Try ``b`` again. This time | +| 7 | ``abcb`` | Try ``b`` again. This time | | | | the character at the | | | | current position is ``'b'``, so | | | | it succeeds. | @@ -255,7 +255,7 @@ is equivalent to ``+``, and ``{0,1}`` is the same as ``?``. It's better to use to read. -Using Regular Expressions +Using regular expressions ========================= Now that we've looked at some simple regular expressions, how do we actually use @@ -264,7 +264,7 @@ expression engine, allowing you to compile REs into objects and then perform matches with them. -Compiling Regular Expressions +Compiling regular expressions ----------------------------- Regular expressions are compiled into pattern objects, which have @@ -295,7 +295,7 @@ disadvantage which is the topic of the next section. .. _the-backslash-plague: -The Backslash Plague +The backslash plague -------------------- As stated earlier, regular expressions use the backslash character (``'\'``) to @@ -335,7 +335,7 @@ expressions will often be written in Python code using this raw string notation. In addition, special escape sequences that are valid in regular expressions, but not valid as Python string literals, now result in a -:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`, +:exc:`SyntaxWarning` and will eventually become a :exc:`SyntaxError`, which means the sequences will be invalid if raw string notation or escaping the backslashes isn't used. @@ -351,7 +351,7 @@ the backslashes isn't used. +-------------------+------------------+ -Performing Matches +Performing matches ------------------ Once you have an object representing a compiled regular expression, what do you @@ -369,10 +369,10 @@ for a complete listing. | | location where this RE matches. | +------------------+-----------------------------------------------+ | ``findall()`` | Find all substrings where the RE matches, and | -| | returns them as a list. | +| | return them as a list. | +------------------+-----------------------------------------------+ | ``finditer()`` | Find all substrings where the RE matches, and | -| | returns them as an :term:`iterator`. | +| | return them as an :term:`iterator`. | +------------------+-----------------------------------------------+ :meth:`~re.Pattern.match` and :meth:`~re.Pattern.search` return ``None`` if no match can be found. If @@ -473,7 +473,7 @@ Two pattern methods return all of the matches for a pattern. The ``r`` prefix, making the literal a raw string literal, is needed in this example because escape sequences in a normal "cooked" string literal that are not recognized by Python, as opposed to regular expressions, now result in a -:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`. See +:exc:`SyntaxWarning` and will eventually become a :exc:`SyntaxError`. See :ref:`the-backslash-plague`. :meth:`~re.Pattern.findall` has to create the entire list before it can be returned as the @@ -491,7 +491,7 @@ result. The :meth:`~re.Pattern.finditer` method returns a sequence of (29, 31) -Module-Level Functions +Module-level functions ---------------------- You don't have to create a pattern object and call its methods; the @@ -518,7 +518,7 @@ Outside of loops, there's not much difference thanks to the internal cache. -Compilation Flags +Compilation flags ----------------- .. currentmodule:: re @@ -642,7 +642,7 @@ of each one. whitespace is in a character class or preceded by an unescaped backslash; this lets you organize and indent the RE more clearly. This flag also lets you put comments within a RE that will be ignored by the engine; comments are marked by - a ``'#'`` that's neither in a character class or preceded by an unescaped + a ``'#'`` that's neither in a character class nor preceded by an unescaped backslash. For example, here's a RE that uses :const:`re.VERBOSE`; see how much easier it @@ -669,7 +669,7 @@ of each one. to understand than the version using :const:`re.VERBOSE`. -More Pattern Power +More pattern power ================== So far we've only covered a part of the features of regular expressions. In @@ -679,7 +679,7 @@ retrieve portions of the text that was matched. .. _more-metacharacters: -More Metacharacters +More metacharacters ------------------- There are some metacharacters that we haven't covered yet. Most of them will be @@ -875,7 +875,7 @@ Backreferences like this aren't often useful for just searching through a string find out that they're *very* useful when performing string substitutions. -Non-capturing and Named Groups +Non-capturing and named groups ------------------------------ Elaborate REs may use many groups, both to capture substrings of interest, and @@ -979,7 +979,7 @@ current point. The regular expression for finding doubled words, 'the the' -Lookahead Assertions +Lookahead assertions -------------------- Another zero-width assertion is the lookahead assertion. Lookahead assertions @@ -1061,7 +1061,7 @@ end in either ``bat`` or ``exe``: ``.*[.](?!bat$|exe$)[^.]*$`` -Modifying Strings +Modifying strings ================= Up to this point, we've simply performed searches against a static string. @@ -1083,7 +1083,7 @@ using the following pattern methods: +------------------+-----------------------------------------------+ -Splitting Strings +Splitting strings ----------------- The :meth:`~re.Pattern.split` method of a pattern splits a string apart @@ -1137,7 +1137,7 @@ argument, but is otherwise the same. :: ['Words', 'words, words.'] -Search and Replace +Search and replace ------------------ Another common task is to find all the matches for a pattern, and replace them @@ -1236,7 +1236,7 @@ pattern object as the first parameter, or use embedded modifiers in the pattern string, e.g. ``sub("(?i)b+", "x", "bbbb BBBB")`` returns ``'x x'``. -Common Problems +Common problems =============== Regular expressions are a powerful tool for some applications, but in some ways @@ -1244,7 +1244,7 @@ their behaviour isn't intuitive and at times they don't behave the way you may expect them to. This section will point out some of the most common pitfalls. -Use String Methods +Use string methods ------------------ Sometimes using the :mod:`re` module is a mistake. If you're matching a fixed @@ -1310,7 +1310,7 @@ string and then backtracking to find a match for the rest of the RE. Use :func:`re.search` instead. -Greedy versus Non-Greedy +Greedy versus non-greedy ------------------------ When repeating a regular expression, as in ``a*``, the resulting action is to @@ -1388,9 +1388,9 @@ Feedback ======== Regular expressions are a complicated topic. Did this document help you -understand them? Were there parts that were unclear, or Problems you +understand them? Were there parts that were unclear, or problems you encountered that weren't covered here? If so, please send suggestions for -improvements to the author. +improvements to the :ref:`issue tracker `. The most complete book on regular expressions is almost certainly Jeffrey Friedl's Mastering Regular Expressions, published by O'Reilly. Unfortunately, diff --git a/Doc/howto/remote_debugging.rst b/Doc/howto/remote_debugging.rst index dfe0176b75a020..1d5cf24d062843 100644 --- a/Doc/howto/remote_debugging.rst +++ b/Doc/howto/remote_debugging.rst @@ -624,3 +624,58 @@ To inject and execute a Python script in a remote process: 6. Set ``_PY_EVAL_PLEASE_STOP_BIT`` in the ``eval_breaker`` field. 7. Resume the process (if suspended). The script will execute at the next safe evaluation point. + +.. _remote-debugging-threat-model: + +Security and threat model +========================= + +The remote debugging protocol relies on the same operating system primitives +used by native debuggers such as GDB and LLDB. Attaching to a process +requires the **same privileges** that those debuggers require, for example +``ptrace`` / Yama LSM on Linux, ``task_for_pid`` on macOS, and +``SeDebugPrivilege`` on Windows. Python does not introduce any new privilege +escalation path; if an attacker already possesses the permissions needed to +attach to a process, they could equally use GDB to read memory or inject +code. + +The following principles define what is, and is not, considered a security +vulnerability in this feature: + +Attaching requires OS-level privileges + On every supported platform the operating system gates cross-process + memory access behind privilege checks (``CAP_SYS_PTRACE``, root, or + administrator rights). A report that demonstrates an issue only after + these privileges have already been obtained is **not** a vulnerability in + CPython, since the OS security boundary was already crossed. + +Crashes or memory errors when reading a compromised process are not vulnerabilities + A tool that reads internal interpreter state from a target process must + trust that memory to be well-formed. If the target process has been + corrupted or is controlled by an attacker, the debugger or profiler may + crash, produce garbage output, or behave unpredictably. This is the same + risk accepted by every ``ptrace``-based debugger. Bugs in this category + (buffer overflows, segmentation faults, or undefined behaviour triggered + by reading corrupted state) are **not** treated as security issues, though + fixes that improve robustness are welcome. + +Vulnerabilities in the target process are not in scope + If the Python process being debugged has already been compromised, the + attacker already controls execution in that process. Demonstrating further + impact from that starting point does not constitute a vulnerability in the + remote debugging protocol. + +When to use ``PYTHON_DISABLE_REMOTE_DEBUG`` +------------------------------------------- + +The environment variable :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` (and the +equivalent :option:`-X disable_remote_debug` flag) allows operators to disable +the in-process side of the protocol as a **defence-in-depth** measure. This +may be useful in hardened or sandboxed deployment environments where no +debugging or profiling of the process is expected and reducing attack surface +is a priority, even though the OS-level privilege checks already prevent +unprivileged access. + +Setting this variable does **not** affect other OS-level debugging interfaces +(``ptrace``, ``/proc``, ``task_for_pid``, etc.), which remain available +according to their own permission models. diff --git a/Doc/includes/capi-extension/spammodule-01.c b/Doc/includes/capi-extension/spammodule-01.c index ac96f17f04712c..0bc34ef57445cb 100644 --- a/Doc/includes/capi-extension/spammodule-01.c +++ b/Doc/includes/capi-extension/spammodule-01.c @@ -35,7 +35,10 @@ static PyMethodDef spam_methods[] = { /// Module slot table +PyABIInfo_VAR(abi_info); + static PyModuleDef_Slot spam_slots[] = { + {Py_mod_abi, &abi_info}, {Py_mod_name, "spam"}, {Py_mod_doc, "A wonderful module with an example function"}, {Py_mod_methods, spam_methods}, diff --git a/Doc/includes/diff.py b/Doc/includes/diff.py index 001619f5f83fc0..bc4bd58ff3e3f1 100644 --- a/Doc/includes/diff.py +++ b/Doc/includes/diff.py @@ -1,4 +1,4 @@ -""" Command line interface to difflib.py providing diffs in four formats: +""" Command-line interface to difflib.py providing diffs in four formats: * ndiff: lists every line and highlights interline changes. * context: highlights clusters of changes in a before/after format. @@ -8,11 +8,11 @@ """ import sys, os, difflib, argparse -from datetime import datetime, timezone +import datetime as dt def file_mtime(path): - t = datetime.fromtimestamp(os.stat(path).st_mtime, - timezone.utc) + t = dt.datetime.fromtimestamp(os.stat(path).st_mtime, + dt.timezone.utc) return t.astimezone().isoformat() def main(): diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 5a463ee9821d61..8ba11b7d12d552 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -1118,7 +1118,15 @@ User defined functions can be used as well: The :func:`bool` function is not recommended as a type converter. All it does is convert empty strings to ``False`` and non-empty strings to ``True``. -This is usually not what is desired. +This is usually not what is desired:: + + >>> parser = argparse.ArgumentParser() + >>> _ = parser.add_argument('--verbose', type=bool) + >>> parser.parse_args(['--verbose', 'False']) + Namespace(verbose=True) + +See :class:`BooleanOptionalAction` or ``action='store_true'`` for common +alternatives. In general, the ``type`` keyword is a convenience that should only be used for simple conversions that can only raise one of the three supported exceptions. diff --git a/Doc/library/array.rst b/Doc/library/array.rst index 5592bd7089ba49..8b0a0c84548127 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -9,7 +9,7 @@ -------------- This module defines an object type which can compactly represent an array of -basic values: characters, integers, floating-point numbers. Arrays are mutable :term:`sequence` +basic values: characters, integers, floating-point numbers, complex numbers. Arrays are mutable :term:`sequence` types and behave very much like lists, except that the type of objects stored in them is constrained. The type is specified at object creation time by using a :dfn:`type code`, which is a single character. The following type codes are @@ -42,10 +42,17 @@ defined: +-----------+--------------------+-------------------+-----------------------+-------+ | ``'Q'`` | unsigned long long | int | 8 | | +-----------+--------------------+-------------------+-----------------------+-------+ +| ``'e'`` | _Float16 | float | 2 | \(3) | ++-----------+--------------------+-------------------+-----------------------+-------+ | ``'f'`` | float | float | 4 | | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'d'`` | double | float | 8 | | +-----------+--------------------+-------------------+-----------------------+-------+ +| ``'F'`` | float complex | complex | 8 | \(4) | ++-----------+--------------------+-------------------+-----------------------+-------+ +| ``'D'`` | double complex | complex | 16 | \(4) | ++-----------+--------------------+-------------------+-----------------------+-------+ + Notes: @@ -63,6 +70,31 @@ Notes: (2) .. versionadded:: 3.13 +(3) + The IEEE 754 binary16 "half precision" type was introduced in the 2008 + revision of the `IEEE 754 standard `_. + This type is not widely supported by C compilers. It's available + as :c:expr:`_Float16` type, if the compiler supports the Annex H + of the C23 standard. + + .. versionadded:: next + +(4) + Complex types (``F`` and ``D``) are available unconditionally, + regardless on support for complex types (the Annex G of the C11 standard) + by the C compiler. + As specified in the C11 standard, each complex type is represented by a + two-element C array containing, respectively, the real and imaginary parts. + + .. versionadded:: 3.15 + +.. seealso:: + + The :ref:`ctypes ` and + :ref:`struct ` modules, + as well as third-party modules like `numpy `__, + use similar -- but slightly different -- type codes. + The actual representation of values is determined by the machine architecture (strictly speaking, by the C implementation). The actual size can be accessed @@ -112,9 +144,9 @@ The module defines the following type: The length in bytes of one array item in the internal representation. - .. method:: append(x) + .. method:: append(value, /) - Append a new item with value *x* to the end of the array. + Append a new item with the specified value to the end of the array. .. method:: buffer_info() @@ -139,17 +171,18 @@ The module defines the following type: .. method:: byteswap() "Byteswap" all items of the array. This is only supported for values which are - 1, 2, 4, or 8 bytes in size; for other types of values, :exc:`RuntimeError` is + 1, 2, 4, 8 or 16 bytes in size; for other types of values, :exc:`RuntimeError` is raised. It is useful when reading data from a file written on a machine with a - different byte order. + different byte order. Note, that for complex types the order of + components (the real part, followed by imaginary part) is preserved. - .. method:: count(x) + .. method:: count(value, /) - Return the number of occurrences of *x* in the array. + Return the number of occurrences of *value* in the array. - .. method:: extend(iterable) + .. method:: extend(iterable, /) Append items from *iterable* to the end of the array. If *iterable* is another array, it must have *exactly* the same type code; if not, :exc:`TypeError` will @@ -157,7 +190,7 @@ The module defines the following type: must be the right type to be appended to the array. - .. method:: frombytes(buffer) + .. method:: frombytes(buffer, /) Appends items from the :term:`bytes-like object`, interpreting its content as an array of machine values (as if it had been read @@ -167,7 +200,7 @@ The module defines the following type: :meth:`!fromstring` is renamed to :meth:`frombytes` for clarity. - .. method:: fromfile(f, n) + .. method:: fromfile(f, n, /) Read *n* items (as machine values) from the :term:`file object` *f* and append them to the end of the array. If less than *n* items are available, @@ -175,13 +208,13 @@ The module defines the following type: inserted into the array. - .. method:: fromlist(list) + .. method:: fromlist(list, /) Append items from the list. This is equivalent to ``for x in list: a.append(x)`` except that if there is a type error, the array is unchanged. - .. method:: fromunicode(s) + .. method:: fromunicode(ustr, /) Extends this array with data from the given Unicode string. The array must have type code ``'u'`` or ``'w'``; otherwise a :exc:`ValueError` is raised. @@ -189,33 +222,33 @@ The module defines the following type: array of some other type. - .. method:: index(x[, start[, stop]]) + .. method:: index(value[, start[, stop]]) Return the smallest *i* such that *i* is the index of the first occurrence of - *x* in the array. The optional arguments *start* and *stop* can be - specified to search for *x* within a subsection of the array. Raise - :exc:`ValueError` if *x* is not found. + *value* in the array. The optional arguments *start* and *stop* can be + specified to search for *value* within a subsection of the array. Raise + :exc:`ValueError` if *value* is not found. .. versionchanged:: 3.10 Added optional *start* and *stop* parameters. - .. method:: insert(i, x) + .. method:: insert(index, value, /) - Insert a new item with value *x* in the array before position *i*. Negative + Insert a new item *value* in the array before position *index*. Negative values are treated as being relative to the end of the array. - .. method:: pop([i]) + .. method:: pop(index=-1, /) Removes the item with the index *i* from the array and returns it. The optional argument defaults to ``-1``, so that by default the last item is removed and returned. - .. method:: remove(x) + .. method:: remove(value, /) - Remove the first occurrence of *x* from the array. + Remove the first occurrence of *value* from the array. .. method:: clear() @@ -240,7 +273,7 @@ The module defines the following type: :meth:`!tostring` is renamed to :meth:`tobytes` for clarity. - .. method:: tofile(f) + .. method:: tofile(f, /) Write all items (as machine values) to the :term:`file object` *f*. @@ -282,3 +315,5 @@ Examples:: `NumPy `_ The NumPy package defines another array type. + +.. _ieee 754 standard: https://en.wikipedia.org/wiki/IEEE_754-2008_revision diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 9660ad70932764..9b4e7ae18348f1 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -131,6 +131,14 @@ Node classes Simple indices are represented by their value, extended slices are represented as tuples. +.. versionchanged:: 3.13 + + AST node constructors were changed to provide sensible defaults for omitted + fields: optional fields now default to ``None``, list fields default to an + empty list, and fields of type :class:`!ast.expr_context` default to + :class:`Load() `. Previously, omitted attributes would not exist on constructed + nodes (accessing them raised :exc:`AttributeError`). + .. versionchanged:: 3.14 The :meth:`~object.__repr__` output of :class:`~ast.AST` nodes includes diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index 7831b613bd4a60..f3409bcd2df648 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -248,3 +248,225 @@ Output in debug mode:: File "../t.py", line 4, in bug raise Exception("not consumed") Exception: not consumed + + +Asynchronous generators best practices +====================================== + +Writing correct and efficient asyncio code requires awareness of certain pitfalls. +This section outlines essential best practices that can save you hours of debugging. + + +Close asynchronous generators explicitly +---------------------------------------- + +It is recommended to manually close the +:term:`asynchronous generator `. If a generator +exits early - for example, due to an exception raised in the body of +an ``async for`` loop - its asynchronous cleanup code may run in an +unexpected context. This can occur after the tasks it depends on have completed, +or during the event loop shutdown when the async-generator's garbage collection +hook is called. + +To avoid this, explicitly close the generator by calling its +:meth:`~agen.aclose` method, or use the :func:`contextlib.aclosing` +context manager:: + + import asyncio + import contextlib + + async def gen(): + yield 1 + yield 2 + + async def func(): + async with contextlib.aclosing(gen()) as g: + async for x in g: + break # Don't iterate until the end + + asyncio.run(func()) + +As noted above, the cleanup code for these asynchronous generators is deferred. +The following example demonstrates that the finalization of an asynchronous +generator can occur in an unexpected order:: + + import asyncio + work_done = False + + async def cursor(): + try: + yield 1 + finally: + assert work_done + + async def rows(): + global work_done + try: + yield 2 + finally: + await asyncio.sleep(0.1) # immitate some async work + work_done = True + + + async def main(): + async for c in cursor(): + async for r in rows(): + break + break + + asyncio.run(main()) + +For this example, we get the following output:: + + unhandled exception during asyncio.run() shutdown + task: ()> exception=AssertionError()> + Traceback (most recent call last): + File "example.py", line 6, in cursor + yield 1 + asyncio.exceptions.CancelledError + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "example.py", line 8, in cursor + assert work_done + ^^^^^^^^^ + AssertionError + +The ``cursor()`` asynchronous generator was finalized before the ``rows`` +generator - an unexpected behavior. + +The example can be fixed by explicitly closing the +``cursor`` and ``rows`` async-generators:: + + async def main(): + async with contextlib.aclosing(cursor()) as cursor_gen: + async for c in cursor_gen: + async with contextlib.aclosing(rows()) as rows_gen: + async for r in rows_gen: + break + break + + +Create asynchronous generators only when the event loop is running +------------------------------------------------------------------ + +It is recommended to create +:term:`asynchronous generators ` only after +the event loop has been created. + +To ensure that asynchronous generators close reliably, the event loop uses the +:func:`sys.set_asyncgen_hooks` function to register callback functions. These +callbacks update the list of running asynchronous generators to keep it in a +consistent state. + +When the :meth:`loop.shutdown_asyncgens() ` +function is called, the running generators are stopped gracefully and the +list is cleared. + +The asynchronous generator invokes the corresponding system hook during its +first iteration. At the same time, the generator records that the hook has +been called and does not call it again. + +Therefore, if iteration begins before the event loop is created, +the event loop will not be able to add the generator to its list of active +generators because the hooks are set after the generator attempts to call them. +Consequently, the event loop will not be able to terminate the generator +if necessary. + +Consider the following example:: + + import asyncio + + async def agenfn(): + try: + yield 10 + finally: + await asyncio.sleep(0) + + + with asyncio.Runner() as runner: + agen = agenfn() + print(runner.run(anext(agen))) + del agen + +Output:: + + 10 + Exception ignored while closing generator : + Traceback (most recent call last): + File "example.py", line 13, in + del agen + ^^^^ + RuntimeError: async generator ignored GeneratorExit + +This example can be fixed as follows:: + + import asyncio + + async def agenfn(): + try: + yield 10 + finally: + await asyncio.sleep(0) + + async def main(): + agen = agenfn() + print(await anext(agen)) + del agen + + asyncio.run(main()) + + +Avoid concurrent iteration and closure of the same generator +------------------------------------------------------------ + +Async generators may be reentered while another +:meth:`~agen.__anext__` / :meth:`~agen.athrow` / :meth:`~agen.aclose` call is in +progress. This may lead to an inconsistent state of the async generator and can +cause errors. + +Let's consider the following example:: + + import asyncio + + async def consumer(): + for idx in range(100): + await asyncio.sleep(0) + message = yield idx + print('received', message) + + async def amain(): + agenerator = consumer() + await agenerator.asend(None) + + fa = asyncio.create_task(agenerator.asend('A')) + fb = asyncio.create_task(agenerator.asend('B')) + await fa + await fb + + asyncio.run(amain()) + +Output:: + + received A + Traceback (most recent call last): + File "test.py", line 38, in + asyncio.run(amain()) + ~~~~~~~~~~~^^^^^^^^^ + File "Lib/asyncio/runners.py", line 204, in run + return runner.run(main) + ~~~~~~~~~~^^^^^^ + File "Lib/asyncio/runners.py", line 127, in run + return self._loop.run_until_complete(task) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ + File "Lib/asyncio/base_events.py", line 719, in run_until_complete + return future.result() + ~~~~~~~~~~~~~^^ + File "test.py", line 36, in amain + await fb + RuntimeError: anext(): asynchronous generator is already running + + +Therefore, it is recommended to avoid using asynchronous generators in parallel +tasks or across multiple event loops. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index bdb24b3a58c267..79c9516cda2d60 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -4,7 +4,7 @@ .. _asyncio-event-loop: ========== -Event Loop +Event loop ========== **Source code:** :source:`Lib/asyncio/events.py`, @@ -105,7 +105,7 @@ This documentation page contains the following sections: .. _asyncio-event-loop-methods: -Event Loop Methods +Event loop methods ================== Event loops have **low-level** APIs for the following: @@ -361,7 +361,7 @@ clocks to track time. The :func:`asyncio.sleep` function. -Creating Futures and Tasks +Creating futures and tasks ^^^^^^^^^^^^^^^^^^^^^^^^^^ .. method:: loop.create_future() @@ -962,7 +962,7 @@ Transferring files .. versionadded:: 3.7 -TLS Upgrade +TLS upgrade ^^^^^^^^^^^ .. method:: loop.start_tls(transport, protocol, \ @@ -1431,7 +1431,7 @@ Executing code in thread or process pools :class:`~concurrent.futures.ThreadPoolExecutor`. -Error Handling API +Error handling API ^^^^^^^^^^^^^^^^^^ Allows customizing how exceptions are handled in the event loop. @@ -1534,7 +1534,7 @@ Enabling debug mode The :ref:`debug mode of asyncio `. -Running Subprocesses +Running subprocesses ^^^^^^^^^^^^^^^^^^^^ Methods described in this subsections are low-level. In regular @@ -1672,7 +1672,7 @@ async/await code consider using the high-level are going to be used to construct shell commands. -Callback Handles +Callback handles ================ .. class:: Handle @@ -1715,7 +1715,7 @@ Callback Handles .. versionadded:: 3.7 -Server Objects +Server objects ============== Server objects are created by :meth:`loop.create_server`, @@ -1858,7 +1858,7 @@ Do not instantiate the :class:`Server` class directly. .. _asyncio-event-loops: .. _asyncio-event-loop-implementations: -Event Loop Implementations +Event loop implementations ========================== asyncio ships with two different event loop implementations: @@ -1971,10 +1971,10 @@ callback uses the :meth:`loop.call_later` method to reschedule itself after 5 seconds, and then stops the event loop:: import asyncio - import datetime + import datetime as dt def display_date(end_time, loop): - print(datetime.datetime.now()) + print(dt.datetime.now()) if (loop.time() + 1.0) < end_time: loop.call_later(1, display_date, end_time, loop) else: @@ -2055,7 +2055,7 @@ Wait until a file descriptor received some data using the Set signal handlers for SIGINT and SIGTERM ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -(This ``signals`` example only works on Unix.) +(This ``signal`` example only works on Unix.) Register handlers for signals :const:`~signal.SIGINT` and :const:`~signal.SIGTERM` using the :meth:`loop.add_signal_handler` method:: diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 5208f14c94a50f..58f77feb311984 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -1037,7 +1037,7 @@ The subprocess is created by the :meth:`loop.subprocess_exec` method:: # low-level APIs. loop = asyncio.get_running_loop() - code = 'import datetime; print(datetime.datetime.now())' + code = 'import datetime as dt; print(dt.datetime.now())' exit_future = asyncio.Future(loop=loop) # Create the subprocess controlled by DateProtocol; diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index 9416c758e51d95..a6514649bf9a0a 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -311,8 +311,16 @@ their completion. A ``None`` value indicates that the process has not terminated yet. - A negative value ``-N`` indicates that the child was terminated - by signal ``N`` (POSIX only). + For processes created with :func:`~asyncio.create_subprocess_exec`, a negative + value ``-N`` indicates that the child was terminated by signal ``N`` + (POSIX only). + + For processes created with :func:`~asyncio.create_subprocess_shell`, the + return code reflects the exit status of the shell itself (e.g. ``/bin/sh``), + which may map signals to codes such as ``128+N``. See the + documentation of the shell (for example, the Bash manual's Exit Status) + for details. + .. _asyncio-subprocess-threads: @@ -351,7 +359,7 @@ function:: import sys async def get_date(): - code = 'import datetime; print(datetime.datetime.now())' + code = 'import datetime as dt; print(dt.datetime.now())' # Create the subprocess; redirect the standard output # into a pipe. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index e2a6752be12b67..4e60eee44290af 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -2,7 +2,7 @@ ==================== -Coroutines and Tasks +Coroutines and tasks ==================== This section outlines high-level asyncio APIs to work with coroutines @@ -231,7 +231,7 @@ A good example of a low-level function that returns a Future object is :meth:`loop.run_in_executor`. -Creating Tasks +Creating tasks ============== **Source code:** :source:`Lib/asyncio/tasks.py` @@ -300,7 +300,7 @@ Creating Tasks Added the *eager_start* parameter by passing on all *kwargs*. -Task Cancellation +Task cancellation ================= Tasks can easily and safely be cancelled. @@ -324,7 +324,7 @@ remove the cancellation state. .. _taskgroups: -Task Groups +Task groups =========== Task groups combine a task creation API with a convenient @@ -427,7 +427,7 @@ reported by :meth:`asyncio.Task.cancelling`. Improved handling of simultaneous internal and external cancellations and correct preservation of cancellation counts. -Terminating a Task Group +Terminating a task group ------------------------ While terminating a task group is not natively supported by the standard @@ -498,13 +498,13 @@ Sleeping for 5 seconds:: import asyncio - import datetime + import datetime as dt async def display_date(): loop = asyncio.get_running_loop() end_time = loop.time() + 5.0 while True: - print(datetime.datetime.now()) + print(dt.datetime.now()) if (loop.time() + 1.0) >= end_time: break await asyncio.sleep(1) @@ -519,7 +519,7 @@ Sleeping Raises :exc:`ValueError` if *delay* is :data:`~math.nan`. -Running Tasks Concurrently +Running tasks concurrently ========================== .. awaitablefunction:: gather(*aws, return_exceptions=False) @@ -621,7 +621,7 @@ Running Tasks Concurrently .. _eager-task-factory: -Eager Task Factory +Eager task factory ================== .. function:: eager_task_factory(loop, coro, *, name=None, context=None) @@ -664,7 +664,7 @@ Eager Task Factory .. versionadded:: 3.12 -Shielding From Cancellation +Shielding from cancellation =========================== .. awaitablefunction:: shield(aw) @@ -894,7 +894,7 @@ Timeouts Raises :exc:`TimeoutError` instead of :exc:`asyncio.TimeoutError`. -Waiting Primitives +Waiting primitives ================== .. function:: wait(aws, *, timeout=None, return_when=ALL_COMPLETED) @@ -1014,7 +1014,7 @@ Waiting Primitives or as a plain :term:`iterator` (previously it was only a plain iterator). -Running in Threads +Running in threads ================== .. function:: to_thread(func, /, *args, **kwargs) @@ -1074,7 +1074,7 @@ Running in Threads .. versionadded:: 3.9 -Scheduling From Other Threads +Scheduling from other threads ============================= .. function:: run_coroutine_threadsafe(coro, loop) @@ -1198,7 +1198,7 @@ Introspection .. _asyncio-task-obj: -Task Object +Task object =========== .. class:: Task(coro, *, loop=None, name=None, context=None, eager_start=False) diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 771628677c3d98..425dff8f2a9ad1 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -51,7 +51,7 @@ The :rfc:`4648` encodings are suitable for encoding binary data so that it can b safely sent by email, used as parts of URLs, or included as part of an HTTP POST request. -.. function:: b64encode(s, altchars=None, *, wrapcol=0) +.. function:: b64encode(s, altchars=None, *, padded=True, wrapcol=0) Encode the :term:`bytes-like object` *s* using Base64 and return the encoded :class:`bytes`. @@ -61,6 +61,10 @@ POST request. This allows an application to e.g. generate URL or filesystem safe Base64 strings. The default is ``None``, for which the standard Base64 alphabet is used. + If *padded* is true (default), pad the encoded data with the '=' + character to a size multiple of 4. + If *padded* is false, do not add the pad characters. + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character after at most every *wrapcol* characters. If *wrapcol* is zero (default), do not insert any newlines. @@ -69,11 +73,11 @@ POST request. :exc:`TypeError` if *altchars* is not a :term:`bytes-like object`. .. versionchanged:: 3.15 - Added the *wrapcol* parameter. + Added the *padded* and *wrapcol* parameters. -.. function:: b64decode(s, altchars=None, validate=False) - b64decode(s, altchars=None, validate=True, *, ignorechars) +.. function:: b64decode(s, altchars=None, validate=False, *, padded=True) + b64decode(s, altchars=None, validate=True, *, ignorechars, padded=True) Decode the Base64 encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. @@ -82,6 +86,11 @@ POST request. of length 2 which specifies the alternative alphabet used instead of the ``+`` and ``/`` characters. + If *padded* is true, the last group of 4 base 64 alphabet characters must + be padded with the '=' character. + If *padded* is false, the '=' character is treated as other non-alphabet + characters (depending on the value of *validate* and *ignorechars*). + A :exc:`binascii.Error` exception is raised if *s* is incorrectly padded. @@ -106,7 +115,7 @@ POST request. For more information about the strict base64 check, see :func:`binascii.a2b_base64` .. versionchanged:: 3.15 - Added the *ignorechars* parameter. + Added the *ignorechars* and *padded* parameters. .. deprecated:: 3.15 Accepting the ``+`` and ``/`` characters with an alternative alphabet @@ -125,16 +134,19 @@ POST request. Base64 alphabet and return the decoded :class:`bytes`. -.. function:: urlsafe_b64encode(s) +.. function:: urlsafe_b64encode(s, *, padded=True) Encode :term:`bytes-like object` *s* using the URL- and filesystem-safe alphabet, which substitutes ``-`` instead of ``+`` and ``_`` instead of ``/`` in the standard Base64 alphabet, and return the encoded :class:`bytes`. The result - can still contain ``=``. + can still contain ``=`` if *padded* is true (default). + .. versionchanged:: next + Added the *padded* parameter. -.. function:: urlsafe_b64decode(s) + +.. function:: urlsafe_b64decode(s, *, padded=False) Decode :term:`bytes-like object` or ASCII string *s* using the URL- and filesystem-safe @@ -142,17 +154,32 @@ POST request. ``/`` in the standard Base64 alphabet, and return the decoded :class:`bytes`. + .. versionchanged:: next + Added the *padded* parameter. + Padding of input is no longer required by default. + .. deprecated:: 3.15 Accepting the ``+`` and ``/`` characters is now deprecated. -.. function:: b32encode(s) +.. function:: b32encode(s, *, padded=True, wrapcol=0) Encode the :term:`bytes-like object` *s* using Base32 and return the encoded :class:`bytes`. + If *padded* is true (default), pad the encoded data with the '=' + character to a size multiple of 8. + If *padded* is false, do not add the pad characters. + + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not add any newlines. + + .. versionchanged:: next + Added the *padded* and *wrapcol* parameters. -.. function:: b32decode(s, casefold=False, map01=None) + +.. function:: b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b'') Decode the Base32 encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. @@ -168,20 +195,34 @@ POST request. digit 0 is always mapped to the letter O). For security purposes the default is ``None``, so that 0 and 1 are not allowed in the input. + If *padded* is true, the last group of 8 base 32 alphabet characters must + be padded with the '=' character. + If *padded* is false, the '=' character is treated as other non-alphabet + characters (depending on the value of *ignorechars*). + + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + A :exc:`binascii.Error` is raised if *s* is incorrectly padded or if there are non-alphabet characters present in the input. + .. versionchanged:: next + Added the *ignorechars* and *padded* parameters. + -.. function:: b32hexencode(s) +.. function:: b32hexencode(s, *, padded=True, wrapcol=0) Similar to :func:`b32encode` but uses the Extended Hex Alphabet, as defined in :rfc:`4648`. .. versionadded:: 3.10 + .. versionchanged:: next + Added the *padded* and *wrapcol* parameters. -.. function:: b32hexdecode(s, casefold=False) + +.. function:: b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b'') Similar to :func:`b32decode` but uses the Extended Hex Alphabet, as defined in :rfc:`4648`. @@ -193,14 +234,24 @@ POST request. .. versionadded:: 3.10 + .. versionchanged:: next + Added the *ignorechars* and *padded* parameters. + -.. function:: b16encode(s) +.. function:: b16encode(s, *, wrapcol=0) Encode the :term:`bytes-like object` *s* using Base16 and return the encoded :class:`bytes`. + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not add any newlines. + + .. versionchanged:: next + Added the *wrapcol* parameter. + -.. function:: b16decode(s, casefold=False) +.. function:: b16decode(s, casefold=False, *, ignorechars=b'') Decode the Base16 encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. @@ -209,10 +260,17 @@ POST request. lowercase alphabet is acceptable as input. For security purposes, the default is ``False``. + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + A :exc:`binascii.Error` is raised if *s* is incorrectly padded or if there are non-alphabet characters present in the input. + .. versionchanged:: next + Added the *ignorechars* parameter. + + .. _base64-base-85: Base85 Encodings @@ -277,7 +335,7 @@ Refer to the documentation of the individual functions for more information. .. versionadded:: 3.4 -.. function:: b85encode(b, pad=False) +.. function:: b85encode(b, pad=False, *, wrapcol=0) Encode the :term:`bytes-like object` *b* using base85 (as used in e.g. git-style binary diffs) and return the encoded :class:`bytes`. @@ -285,19 +343,32 @@ Refer to the documentation of the individual functions for more information. If *pad* is true, the input is padded with ``b'\0'`` so its length is a multiple of 4 bytes before encoding. + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not add any newlines. + .. versionadded:: 3.4 + .. versionchanged:: next + Added the *wrapcol* parameter. + -.. function:: b85decode(b) +.. function:: b85decode(b, *, ignorechars=b'') Decode the base85-encoded :term:`bytes-like object` or ASCII string *b* and return the decoded :class:`bytes`. Padding is implicitly removed, if necessary. + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + .. versionadded:: 3.4 + .. versionchanged:: next + Added the *ignorechars* parameter. + -.. function:: z85encode(s, pad=False) +.. function:: z85encode(s, pad=False, *, wrapcol=0) Encode the :term:`bytes-like object` *s* using Z85 (as used in ZeroMQ) and return the encoded :class:`bytes`. See `Z85 specification @@ -306,20 +377,33 @@ Refer to the documentation of the individual functions for more information. If *pad* is true, the input is padded with ``b'\0'`` so its length is a multiple of 4 bytes before encoding. + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not add any newlines. + .. versionadded:: 3.13 .. versionchanged:: 3.15 The *pad* parameter was added. + .. versionchanged:: next + Added the *wrapcol* parameter. + -.. function:: z85decode(s) +.. function:: z85decode(s, *, ignorechars=b'') Decode the Z85-encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. See `Z85 specification `_ for more information. + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + .. versionadded:: 3.13 + .. versionchanged:: next + Added the *ignorechars* parameter. + .. _base64-legacy: diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 70ba036756ff32..4f2edb7eff8a8f 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -48,8 +48,8 @@ The :mod:`!binascii` module defines the following functions: Added the *backtick* parameter. -.. function:: a2b_base64(string, /, *, alphabet=BASE64_ALPHABET, strict_mode=False) - a2b_base64(string, /, *, ignorechars, alphabet=BASE64_ALPHABET, strict_mode=True) +.. function:: a2b_base64(string, /, *, padded=True, alphabet=BASE64_ALPHABET, strict_mode=False) + a2b_base64(string, /, *, ignorechars, padded=True, alphabet=BASE64_ALPHABET, strict_mode=True) Convert a block of base64 data back to binary and return the binary data. More than one line may be passed at a time. @@ -57,6 +57,11 @@ The :mod:`!binascii` module defines the following functions: Optional *alphabet* must be a :class:`bytes` object of length 64 which specifies an alternative alphabet. + If *padded* is true, the last group of 4 base 64 alphabet characters must + be padded with the '=' character. + If *padded* is false, the '=' character is treated as other non-alphabet + characters (depending on the value of *strict_mode* and *ignorechars*). + If *ignorechars* is specified, it should be a :term:`bytes-like object` containing characters to ignore from the input when *strict_mode* is true. If *ignorechars* contains the pad character ``'='``, the pad characters @@ -79,14 +84,18 @@ The :mod:`!binascii` module defines the following functions: Added the *strict_mode* parameter. .. versionchanged:: 3.15 - Added the *alphabet* and *ignorechars* parameters. + Added the *alphabet*, *ignorechars* and *padded* parameters. -.. function:: b2a_base64(data, *, alphabet=BASE64_ALPHABET, wrapcol=0, newline=True) +.. function:: b2a_base64(data, *, padded=True, alphabet=BASE64_ALPHABET, wrapcol=0, newline=True) Convert binary data to a line(s) of ASCII characters in base64 coding, as specified in :rfc:`4648`. + If *padded* is true (default), pad the encoded data with the '=' + character to a size multiple of 4. + If *padded* is false, do not add the pad characters. + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character after at most every *wrapcol* characters. If *wrapcol* is zero (default), do not insert any newlines. @@ -98,10 +107,10 @@ The :mod:`!binascii` module defines the following functions: Added the *newline* parameter. .. versionchanged:: 3.15 - Added the *alphabet* and *wrapcol* parameters. + Added the *alphabet*, *padded* and *wrapcol* parameters. -.. function:: a2b_ascii85(string, /, *, foldspaces=False, adobe=False, ignorechars=b"") +.. function:: a2b_ascii85(string, /, *, foldspaces=False, adobe=False, ignorechars=b'') Convert Ascii85 data back to binary and return the binary data. @@ -151,7 +160,7 @@ The :mod:`!binascii` module defines the following functions: .. versionadded:: 3.15 -.. function:: a2b_base85(string, /, *, alphabet=BASE85_ALPHABET) +.. function:: a2b_base85(string, /, *, alphabet=BASE85_ALPHABET, ignorechars=b'') Convert Base85 data back to binary and return the binary data. More than one line may be passed at a time. @@ -164,12 +173,15 @@ The :mod:`!binascii` module defines the following functions: Optional *alphabet* must be a :class:`bytes` object of length 85 which specifies an alternative alphabet. + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + Invalid Base85 data will raise :exc:`binascii.Error`. .. versionadded:: 3.15 -.. function:: b2a_base85(data, /, *, alphabet=BASE85_ALPHABET, pad=False) +.. function:: b2a_base85(data, /, *, alphabet=BASE85_ALPHABET, wrapcol=0, pad=False) Convert binary data to a line of ASCII characters in Base85 coding. The return value is the converted line. @@ -177,12 +189,67 @@ The :mod:`!binascii` module defines the following functions: Optional *alphabet* must be a :term:`bytes-like object` of length 85 which specifies an alternative alphabet. + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. + If *pad* is true, the input is padded with ``b'\0'`` so its length is a multiple of 4 bytes before encoding. .. versionadded:: 3.15 +.. function:: a2b_base32(string, /, *, padded=True, alphabet=BASE32_ALPHABET, ignorechars=b'') + + Convert base32 data back to binary and return the binary data. + + Valid base32 data contains characters from the base32 alphabet specified + in :rfc:`4648` in groups of eight (if necessary, the final group is padded + to eight characters with ``=``). Each group encodes 40 bits of binary data + in the range from ``0`` to ``2 ** 40 - 1``, inclusive. + + .. note:: + This function does not map lowercase characters (which are invalid in + standard base32) to their uppercase counterparts, nor does it + contextually map ``0`` to ``O`` and ``1`` to ``I``/``L`` as :rfc:`4648` + allows. + + Optional *alphabet* must be a :class:`bytes` object of length 32 which + specifies an alternative alphabet. + + If *padded* is true, the last group of 8 base 32 alphabet characters must + be padded with the '=' character. + If *padded* is false, the '=' character is treated as other non-alphabet + characters (depending on the value of *ignorechars*). + + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + If *ignorechars* contains the pad character ``'='``, the pad characters + presented before the end of the encoded data and the excess pad characters + will be ignored. + + Invalid base32 data will raise :exc:`binascii.Error`. + + .. versionadded:: next + +.. function:: b2a_base32(data, /, *, padded=True, alphabet=BASE32_ALPHABET, wrapcol=0) + + Convert binary data to a line of ASCII characters in base32 coding, + as specified in :rfc:`4648`. The return value is the converted line. + + Optional *alphabet* must be a :term:`bytes-like object` of length 32 which + specifies an alternative alphabet. + + If *padded* is true (default), pad the encoded data with the '=' + character to a size multiple of 8. + If *padded* is false, do not add the pad characters. + + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. + + .. versionadded:: next + .. function:: a2b_qp(data, header=False) Convert a block of quoted-printable data back to binary and return the binary @@ -256,18 +323,25 @@ The :mod:`!binascii` module defines the following functions: .. versionchanged:: 3.8 The *sep* and *bytes_per_sep* parameters were added. -.. function:: a2b_hex(hexstr) - unhexlify(hexstr) +.. function:: a2b_hex(hexstr, *, ignorechars=b'') + unhexlify(hexstr, *, ignorechars=b'') Return the binary data represented by the hexadecimal string *hexstr*. This function is the inverse of :func:`b2a_hex`. *hexstr* must contain an even number of hexadecimal digits (which can be upper or lower case), otherwise an :exc:`Error` exception is raised. + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + Similar functionality (accepting only text string arguments, but more liberal towards whitespace) is also accessible using the :meth:`bytes.fromhex` class method. + .. versionchanged:: next + Added the *ignorechars* parameter. + + .. exception:: Error Exception raised on errors. These are usually programming errors. @@ -327,6 +401,20 @@ The :mod:`!binascii` module defines the following functions: .. versionadded:: next +.. data:: BASE32_ALPHABET + + The Base 32 alphabet according to :rfc:`4648`. + + .. versionadded:: next + +.. data:: BASE32HEX_ALPHABET + + The "Extended Hex" Base 32 alphabet according to :rfc:`4648`. + Data encoded with this alphabet maintains its sort order during bitwise + comparisons. + + .. versionadded:: next + .. seealso:: diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 58bbc9afe709af..c8dcaef80188cd 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -237,7 +237,9 @@ For example:: [('the', 1143), ('and', 966), ('to', 762), ('of', 669), ('i', 631), ('you', 554), ('a', 546), ('my', 514), ('hamlet', 471), ('in', 451)] -.. class:: Counter([iterable-or-mapping]) +.. class:: Counter(**kwargs) + Counter(iterable, /, **kwargs) + Counter(mapping, /, **kwargs) A :class:`Counter` is a :class:`dict` subclass for counting :term:`hashable` objects. It is a collection where elements are stored as dictionary keys @@ -287,7 +289,7 @@ For example:: >>> sorted(c.elements()) ['a', 'a', 'a', 'a', 'b', 'b'] - .. method:: most_common([n]) + .. method:: most_common(n=None) Return a list of the *n* most common elements and their counts from the most common to the least. If *n* is omitted or ``None``, @@ -297,7 +299,9 @@ For example:: >>> Counter('abracadabra').most_common(3) [('a', 5), ('b', 2), ('r', 2)] - .. method:: subtract([iterable-or-mapping]) + .. method:: subtract(**kwargs) + subtract(iterable, /, **kwargs) + subtract(mapping, /, **kwargs) Elements are subtracted from an *iterable* or from another *mapping* (or counter). Like :meth:`dict.update` but subtracts counts instead @@ -328,7 +332,9 @@ For example:: This class method is not implemented for :class:`Counter` objects. - .. method:: update([iterable-or-mapping]) + .. method:: update(**kwargs) + update(iterable, /, **kwargs) + update(mapping, /, **kwargs) Elements are counted from an *iterable* or added-in from another *mapping* (or counter). Like :meth:`dict.update` but adds counts @@ -481,14 +487,14 @@ or subtracting from an empty counter. Deque objects support the following methods: - .. method:: append(x) + .. method:: append(item, /) - Add *x* to the right side of the deque. + Add *item* to the right side of the deque. - .. method:: appendleft(x) + .. method:: appendleft(item, /) - Add *x* to the left side of the deque. + Add *item* to the left side of the deque. .. method:: clear() @@ -503,38 +509,38 @@ or subtracting from an empty counter. .. versionadded:: 3.5 - .. method:: count(x) + .. method:: count(value, /) - Count the number of deque elements equal to *x*. + Count the number of deque elements equal to *value*. .. versionadded:: 3.2 - .. method:: extend(iterable) + .. method:: extend(iterable, /) Extend the right side of the deque by appending elements from the iterable argument. - .. method:: extendleft(iterable) + .. method:: extendleft(iterable, /) Extend the left side of the deque by appending elements from *iterable*. Note, the series of left appends results in reversing the order of elements in the iterable argument. - .. method:: index(x[, start[, stop]]) + .. method:: index(value[, start[, stop]]) - Return the position of *x* in the deque (at or after index *start* + Return the position of *value* in the deque (at or after index *start* and before index *stop*). Returns the first match or raises :exc:`ValueError` if not found. .. versionadded:: 3.5 - .. method:: insert(i, x) + .. method:: insert(index, value, /) - Insert *x* into the deque at position *i*. + Insert *value* into the deque at position *index*. If the insertion would cause a bounded deque to grow beyond *maxlen*, an :exc:`IndexError` is raised. @@ -554,7 +560,7 @@ or subtracting from an empty counter. elements are present, raises an :exc:`IndexError`. - .. method:: remove(value) + .. method:: remove(value, /) Remove the first occurrence of *value*. If not found, raises a :exc:`ValueError`. @@ -567,7 +573,7 @@ or subtracting from an empty counter. .. versionadded:: 3.2 - .. method:: rotate(n=1) + .. method:: rotate(n=1, /) Rotate the deque *n* steps to the right. If *n* is negative, rotate to the left. @@ -719,7 +725,9 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, :class:`defaultdict` objects ---------------------------- -.. class:: defaultdict(default_factory=None, /, [...]) +.. class:: defaultdict(default_factory=None, /, **kwargs) + defaultdict(default_factory, mapping, /, **kwargs) + defaultdict(default_factory, iterable, /, **kwargs) Return a new dictionary-like object. :class:`defaultdict` is a subclass of the built-in :class:`dict` class. It overrides one method and adds one writable @@ -735,7 +743,7 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, :class:`defaultdict` objects support the following method in addition to the standard :class:`dict` operations: - .. method:: __missing__(key) + .. method:: __missing__(key, /) If the :attr:`default_factory` attribute is ``None``, this raises a :exc:`KeyError` exception with the *key* as argument. @@ -941,7 +949,7 @@ In addition to the methods inherited from tuples, named tuples support three additional methods and two attributes. To prevent conflicts with field names, the method and attribute names start with an underscore. -.. classmethod:: somenamedtuple._make(iterable) +.. classmethod:: somenamedtuple._make(iterable, /) Class method that makes a new instance from an existing sequence or iterable. @@ -1138,7 +1146,9 @@ Some differences from :class:`dict` still remain: * Until Python 3.8, :class:`dict` lacked a :meth:`~object.__reversed__` method. -.. class:: OrderedDict([items]) +.. class:: OrderedDict(**kwargs) + OrderedDict(mapping, /, **kwargs) + OrderedDict(iterable, /, **kwargs) Return an instance of a :class:`dict` subclass that has methods specialized for rearranging dictionary order. @@ -1319,16 +1329,17 @@ subclass directly from :class:`dict`; however, this class can be easier to work with because the underlying dictionary is accessible as an attribute. -.. class:: UserDict([initialdata]) +.. class:: UserDict(**kwargs) + UserDict(mapping, /, **kwargs) + UserDict(iterable, /, **kwargs) Class that simulates a dictionary. The instance's contents are kept in a regular dictionary, which is accessible via the :attr:`data` attribute of - :class:`UserDict` instances. If *initialdata* is provided, :attr:`data` is - initialized with its contents; note that a reference to *initialdata* will not - be kept, allowing it to be used for other purposes. + :class:`!UserDict` instances. If arguments are provided, they are used to + initialize :attr:`data`, like a regular dictionary. In addition to supporting the methods and operations of mappings, - :class:`UserDict` instances provide the following attribute: + :class:`!UserDict` instances provide the following attribute: .. attribute:: data diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index 4c1750de1d3933..4d720176fcc334 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -24,6 +24,11 @@ can be customized by end users easily. This library does *not* interpret or write the value-type prefixes used in the Windows Registry extended version of INI syntax. +.. warning:: + Be cautious when parsing data from untrusted sources. A malicious + INI file may cause the decoder to consume considerable CPU and memory + resources. Limiting the size of data to be parsed is recommended. + .. seealso:: Module :mod:`tomllib` diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 8b213ef1d49932..0b3ad4573f5fcf 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -221,87 +221,164 @@ Fundamental data types :mod:`!ctypes` defines a number of primitive C compatible data types: -+----------------------+------------------------------------------+----------------------------+ -| ctypes type | C type | Python type | -+======================+==========================================+============================+ -| :class:`c_bool` | :c:expr:`_Bool` | bool (1) | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_char` | :c:expr:`char` | 1-character bytes object | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_wchar` | :c:type:`wchar_t` | 1-character string | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_byte` | :c:expr:`char` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ubyte` | :c:expr:`unsigned char` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_short` | :c:expr:`short` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ushort` | :c:expr:`unsigned short` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int` | :c:expr:`int` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int8` | :c:type:`int8_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int16` | :c:type:`int16_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int32` | :c:type:`int32_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int64` | :c:type:`int64_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint` | :c:expr:`unsigned int` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint8` | :c:type:`uint8_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint16` | :c:type:`uint16_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint32` | :c:type:`uint32_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint64` | :c:type:`uint64_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_long` | :c:expr:`long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ulong` | :c:expr:`unsigned long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_longlong` | :c:expr:`__int64` or :c:expr:`long long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ulonglong` | :c:expr:`unsigned __int64` or | int | -| | :c:expr:`unsigned long long` | | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_size_t` | :c:type:`size_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ssize_t` | :c:type:`ssize_t` or | int | -| | :c:expr:`Py_ssize_t` | | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_time_t` | :c:type:`time_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_float` | :c:expr:`float` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_double` | :c:expr:`double` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_longdouble`| :c:expr:`long double` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_char_p` | :c:expr:`char *` (NUL terminated) | bytes object or ``None`` | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_wchar_p` | :c:expr:`wchar_t *` (NUL terminated) | string or ``None`` | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_void_p` | :c:expr:`void *` | int or ``None`` | -+----------------------+------------------------------------------+----------------------------+ - -(1) - The constructor accepts any object with a truth value. +.. list-table:: + :header-rows: 1 + + * - ctypes type + - C type + - Python type + - :py:attr:`~_SimpleCData._type_` + * - :class:`c_bool` + - :c:expr:`_Bool` + - :py:class:`bool` + - ``'?'`` + * - :class:`c_char` + - :c:expr:`char` + - 1-character :py:class:`bytes` + - ``'c'`` + * - :class:`c_wchar` + - :c:type:`wchar_t` + - 1-character :py:class:`str` + - ``'u'`` + * - :class:`c_byte` + - :c:expr:`char` + - :py:class:`int` + - ``'b'`` + * - :class:`c_ubyte` + - :c:expr:`unsigned char` + - :py:class:`int` + - ``'B'`` + * - :class:`c_short` + - :c:expr:`short` + - :py:class:`int` + - ``'h'`` + * - :class:`c_ushort` + - :c:expr:`unsigned short` + - :py:class:`int` + - ``'H'`` + * - :class:`c_int` + - :c:expr:`int` + - :py:class:`int` + - ``'i'`` \* + * - :class:`c_int8` + - :c:type:`int8_t` + - :py:class:`int` + - \* + * - :class:`c_int16` + - :c:type:`int16_t` + - :py:class:`int` + - \* + * - :class:`c_int32` + - :c:type:`int32_t` + - :py:class:`int` + - \* + * - :class:`c_int64` + - :c:type:`int64_t` + - :py:class:`int` + - \* + * - :class:`c_uint` + - :c:expr:`unsigned int` + - :py:class:`int` + - ``'I'`` \* + * - :class:`c_uint8` + - :c:type:`uint8_t` + - :py:class:`int` + - \* + * - :class:`c_uint16` + - :c:type:`uint16_t` + - :py:class:`int` + - \* + * - :class:`c_uint32` + - :c:type:`uint32_t` + - :py:class:`int` + - \* + * - :class:`c_uint64` + - :c:type:`uint64_t` + - :py:class:`int` + - \* + * - :class:`c_long` + - :c:expr:`long` + - :py:class:`int` + - ``'l'`` + * - :class:`c_ulong` + - :c:expr:`unsigned long` + - :py:class:`int` + - ``'L'`` + * - :class:`c_longlong` + - :c:expr:`long long` + - :py:class:`int` + - ``'q'`` \* + * - :class:`c_ulonglong` + - :c:expr:`unsigned long long` + - :py:class:`int` + - ``'Q'`` \* + * - :class:`c_size_t` + - :c:type:`size_t` + - :py:class:`int` + - \* + * - :class:`c_ssize_t` + - :c:type:`Py_ssize_t` + - :py:class:`int` + - \* + * - :class:`c_time_t` + - :c:type:`time_t` + - :py:class:`int` + - \* + * - :class:`c_float` + - :c:expr:`float` + - :py:class:`float` + - ``'f'`` + * - :class:`c_double` + - :c:expr:`double` + - :py:class:`float` + - ``'d'`` + * - :class:`c_longdouble` + - :c:expr:`long double` + - :py:class:`float` + - ``'g'`` \* + * - :class:`c_char_p` + - :c:expr:`char *` (NUL terminated) + - :py:class:`bytes` or ``None`` + - ``'z'`` + * - :class:`c_wchar_p` + - :c:expr:`wchar_t *` (NUL terminated) + - :py:class:`str` or ``None`` + - ``'Z'`` + * - :class:`c_void_p` + - :c:expr:`void *` + - :py:class:`int` or ``None`` + - ``'P'`` + * - :class:`py_object` + - :c:expr:`PyObject *` + - :py:class:`object` + - ``'O'`` + * - :ref:`VARIANT_BOOL ` + - :c:expr:`short int` + - :py:class:`bool` + - ``'v'`` Additionally, if IEC 60559 compatible complex arithmetic (Annex G) is supported in both C and ``libffi``, the following complex types are available: -+----------------------------------+---------------------------------+-----------------+ -| ctypes type | C type | Python type | -+==================================+=================================+=================+ -| :class:`c_float_complex` | :c:expr:`float complex` | complex | -+----------------------------------+---------------------------------+-----------------+ -| :class:`c_double_complex` | :c:expr:`double complex` | complex | -+----------------------------------+---------------------------------+-----------------+ -| :class:`c_longdouble_complex` | :c:expr:`long double complex` | complex | -+----------------------------------+---------------------------------+-----------------+ +.. list-table:: + :header-rows: 1 + + * - ctypes type + - C type + - Python type + - :py:attr:`~_SimpleCData._type_` + * - :class:`c_float_complex` + - :c:expr:`float complex` + - :py:class:`complex` + - ``'F'`` + * - :class:`c_double_complex` + - :c:expr:`double complex` + - :py:class:`complex` + - ``'D'`` + * - :class:`c_longdouble_complex` + - :c:expr:`long double complex` + - :py:class:`complex` + - ``'G'`` All these types can be created by calling them with an optional initializer of @@ -315,6 +392,16 @@ the correct type and value:: c_ushort(65533) >>> +The constructors for numeric types will convert input using +:py:meth:`~object.__bool__`, +:py:meth:`~object.__index__` (for ``int``), +:py:meth:`~object.__float__` or :py:meth:`~object.__complex__`. +This means :py:class:`~ctypes.c_bool` accepts any object with a truth value:: + + >>> empty_list = [] + >>> c_bool(empty_list) + c_bool(False) + Since these types are mutable, their value can also be changed afterwards:: >>> i = c_int(42) @@ -2478,6 +2565,29 @@ Fundamental data types original object return, always a new object is constructed. The same is true for all other ctypes object instances. + Each subclass has a class attribute: + + .. attribute:: _type_ + + Class attribute that contains an internal type code, as a + single-character string. + See :ref:`ctypes-fundamental-data-types` for a summary. + + Types marked \* in the summary may be (or always are) aliases of a + different :class:`_SimpleCData` subclass, and will not necessarily + use the listed type code. + For example, if the platform's :c:expr:`long`, :c:expr:`long long` + and :c:expr:`time_t` C types are the same, then :class:`c_long`, + :class:`c_longlong` and :class:`c_time_t` all refer to a single class, + :class:`c_long`, whose :attr:`_type_` code is ``'l'``. + The ``'L'`` code will be unused. + + .. seealso:: + + The :mod:`array` and :ref:`struct ` modules, + as well as third-party modules like `numpy `__, + use similar -- but slightly different -- type codes. + Fundamental data types, when returned as foreign function call results, or, for example, by retrieving structure field members or array items, are transparently @@ -2599,6 +2709,8 @@ These are the fundamental ctypes data types: Represents the C :c:expr:`signed long long` datatype. The constructor accepts an optional integer initializer; no overflow checking is done. + On platforms where ``sizeof(long long) == sizeof(long)`` it is an alias + to :class:`c_long`. .. class:: c_short @@ -2610,11 +2722,15 @@ These are the fundamental ctypes data types: .. class:: c_size_t Represents the C :c:type:`size_t` datatype. + Usually an alias for another unsigned integer type. .. class:: c_ssize_t - Represents the C :c:type:`ssize_t` datatype. + Represents the :c:type:`Py_ssize_t` datatype. + This is a signed version of :c:type:`size_t`; + that is, the POSIX :c:type:`ssize_t` type. + Usually an alias for another integer type. .. versionadded:: 3.2 @@ -2622,6 +2738,7 @@ These are the fundamental ctypes data types: .. class:: c_time_t Represents the C :c:type:`time_t` datatype. + Usually an alias for another integer type. .. versionadded:: 3.12 @@ -2674,6 +2791,8 @@ These are the fundamental ctypes data types: Represents the C :c:expr:`unsigned long long` datatype. The constructor accepts an optional integer initializer; no overflow checking is done. + On platforms where ``sizeof(long long) == sizeof(long)`` it is an alias + to :class:`c_long`. .. class:: c_ushort @@ -2725,8 +2844,11 @@ These are the fundamental ctypes data types: .. versionchanged:: 3.14 :class:`!py_object` is now a :term:`generic type`. +.. _ctypes-wintypes: + The :mod:`!ctypes.wintypes` module provides quite some other Windows specific -data types, for example :c:type:`!HWND`, :c:type:`!WPARAM`, or :c:type:`!DWORD`. +data types, for example :c:type:`!HWND`, :c:type:`!WPARAM`, +:c:type:`!VARIANT_BOOL` or :c:type:`!DWORD`. Some useful structures like :c:type:`!MSG` or :c:type:`!RECT` are also defined. diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index e56c4f5e7dfbf7..8b812c173b5953 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -362,7 +362,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. .. _sequence-matcher: -SequenceMatcher Objects +SequenceMatcher objects ----------------------- The :class:`SequenceMatcher` class has this constructor: @@ -590,7 +590,7 @@ are always at least as large as :meth:`~SequenceMatcher.ratio`: .. _sequencematcher-examples: -SequenceMatcher Examples +SequenceMatcher examples ------------------------ This example compares two strings, considering blanks to be "junk": @@ -641,7 +641,7 @@ If you want to know how to change the first sequence into the second, use .. _differ-objects: -Differ Objects +Differ objects -------------- Note that :class:`Differ`\ -generated deltas make no claim to be **minimal** @@ -690,7 +690,7 @@ The :class:`Differ` class has this constructor: .. _differ-examples: -Differ Example +Differ example -------------- This example compares two texts. First we set up the texts, sequences of diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 8a8a2edc9e542d..0a1d2a0bac33ab 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -56,7 +56,7 @@ are not normal Python classes. See --------------- -Module Contents +Module contents --------------- :class:`EnumType` @@ -161,7 +161,7 @@ Module Contents --------------- -Data Types +Data types ---------- @@ -341,7 +341,7 @@ Data Types any public methods defined on *self.__class__*:: >>> from enum import Enum - >>> from datetime import date + >>> import datetime as dt >>> class Weekday(Enum): ... MONDAY = 1 ... TUESDAY = 2 @@ -352,7 +352,7 @@ Data Types ... SUNDAY = 7 ... @classmethod ... def today(cls): - ... print('today is %s' % cls(date.today().isoweekday()).name) + ... print(f'today is {cls(dt.date.today().isoweekday()).name}') ... >>> dir(Weekday.SATURDAY) ['__class__', '__doc__', '__eq__', '__hash__', '__module__', 'name', 'today', 'value'] @@ -970,7 +970,7 @@ Supported ``_sunder_`` names --------------- -Utilities and Decorators +Utilities and decorators ------------------------ .. class:: auto diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 483e5b1d8fdba7..e8c4605d0578e2 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1754,7 +1754,7 @@ are always available. They are listed here in alphabetical order. self.age = age def __repr__(self): - return f"Person('{self.name}', {self.age})" + return f"Person({self.name!r}, {self.age!r})" .. function:: reversed(object, /) diff --git a/Doc/library/getpass.rst b/Doc/library/getpass.rst index 1fb34d14d8b007..a6ca230d5e81a8 100644 --- a/Doc/library/getpass.rst +++ b/Doc/library/getpass.rst @@ -39,13 +39,27 @@ The :mod:`!getpass` module provides two functions: On Unix systems, when *echo_char* is set, the terminal will be configured to operate in :manpage:`noncanonical mode `. - In particular, this means that line editing shortcuts such as - :kbd:`Ctrl+U` will not work and may insert unexpected characters into - the input. + Common terminal control characters are supported: + + * :kbd:`Ctrl+A` - Move cursor to beginning of line + * :kbd:`Ctrl+E` - Move cursor to end of line + * :kbd:`Ctrl+K` - Kill (delete) from cursor to end of line + * :kbd:`Ctrl+U` - Kill (delete) entire line + * :kbd:`Ctrl+W` - Erase previous word + * :kbd:`Ctrl+V` - Insert next character literally (quote) + * :kbd:`Backspace`/:kbd:`DEL` - Delete character before cursor + + These shortcuts work by reading the terminal's configured control + character mappings from termios settings. .. versionchanged:: 3.14 Added the *echo_char* parameter for keyboard feedback. + .. versionchanged:: next + When using non-empty *echo_char* on Unix, keyboard shortcuts (including + cursor movement and line editing) are now properly handled using the + terminal's control character configuration. + .. exception:: GetPassWarning A :exc:`UserWarning` subclass issued when password input may be echoed. diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index bd8c3f09cb43f1..cb8b5f0df88d6c 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -287,6 +287,8 @@ instantiation, of which this module provides three different variants: specifying its value. Note that, after the send_header calls are done, :meth:`end_headers` MUST BE called in order to complete the operation. + This method does not reject input containing CRLF sequences. + .. versionchanged:: 3.2 Headers are stored in an internal buffer. @@ -297,6 +299,8 @@ instantiation, of which this module provides three different variants: buffered and sent directly the output stream.If the *message* is not specified, the HTTP message corresponding the response *code* is sent. + This method does not reject *message* containing CRLF sequences. + .. versionadded:: 3.2 .. method:: end_headers() @@ -555,6 +559,11 @@ Security considerations requests, this makes it possible for files outside of the specified directory to be served. +Methods :meth:`BaseHTTPRequestHandler.send_header` and +:meth:`BaseHTTPRequestHandler.send_response_only` assume sanitized input +and does not perform input validation such as checking for the presence of CRLF +sequences. Untrusted input may result in HTTP Header injection attacks. + Earlier versions of Python did not scrub control characters from the log messages emitted to stderr from ``python -m http.server`` or the default :class:`BaseHTTPRequestHandler` ``.log_message`` diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 4a26419e65bee4..72632a8ef53d5b 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -264,7 +264,7 @@ Basic Usage .. function:: load(fp, *, cls=None, object_hook=None, parse_float=None, \ parse_int=None, parse_constant=None, \ - object_pairs_hook=None, **kw) + object_pairs_hook=None, array_hook=None, **kw) Deserialize *fp* to a Python object using the :ref:`JSON-to-Python conversion table `. @@ -301,6 +301,15 @@ Basic Usage Default ``None``. :type object_pairs_hook: :term:`callable` | None + :param array_hook: + If set, a function that is called with the result of + any JSON array literal decoded with as a Python list. + The return value of this function will be used + instead of the :class:`list`. + This feature can be used to implement custom decoders. + Default ``None``. + :type array_hook: :term:`callable` | None + :param parse_float: If set, a function that is called with the string of every JSON float to be decoded. @@ -349,7 +358,10 @@ Basic Usage conversion length limitation ` to help avoid denial of service attacks. -.. function:: loads(s, *, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw) + .. versionchanged:: next + Added the optional *array_hook* parameter. + +.. function:: loads(s, *, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, array_hook=None, **kw) Identical to :func:`load`, but instead of a file-like object, deserialize *s* (a :class:`str`, :class:`bytes` or :class:`bytearray` @@ -367,7 +379,7 @@ Basic Usage Encoders and Decoders --------------------- -.. class:: JSONDecoder(*, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, strict=True, object_pairs_hook=None) +.. class:: JSONDecoder(*, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, strict=True, object_pairs_hook=None, array_hook=None) Simple JSON decoder. @@ -412,6 +424,14 @@ Encoders and Decoders .. versionchanged:: 3.1 Added support for *object_pairs_hook*. + *array_hook* is an optional function that will be called with the + result of every JSON array decoded as a list. The return value of + *array_hook* will be used instead of the :class:`list`. This feature can be + used to implement custom decoders. + + .. versionchanged:: next + Added support for *array_hook*. + *parse_float* is an optional function that will be called with the string of every JSON float to be decoded. By default, this is equivalent to ``float(num_str)``. This can be used to use another datatype or parser for diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index e5fc19f495226e..72140e41675c35 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -18,7 +18,7 @@ and XML plist files. The property list (``.plist``) file format is a simple serialization supporting basic object types, like dictionaries, lists, numbers and strings. Usually the -top level object is a dictionary. +top level object is a dictionary or a frozen dictionary. To write out and to parse a plist file, use the :func:`dump` and :func:`load` functions. @@ -180,7 +180,7 @@ Examples Generating a plist:: - import datetime + import datetime as dt import plistlib pl = dict( @@ -196,7 +196,7 @@ Generating a plist:: ), someData = b"", someMoreData = b"" * 10, - aDate = datetime.datetime.now() + aDate = dt.datetime.now() ) print(plistlib.dumps(pl).decode()) diff --git a/Doc/library/profiling.sampling.rst b/Doc/library/profiling.sampling.rst index d2b7d9669ab07e..a6ce2f30eadb2c 100644 --- a/Doc/library/profiling.sampling.rst +++ b/Doc/library/profiling.sampling.rst @@ -1003,6 +1003,47 @@ at the top indicate functions that consume significant time either directly or through their callees. +Differential flame graphs +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Differential flame graphs compare two profiling runs to highlight where +performance changed. This helps identify regressions introduced by code changes +and validate that optimizations achieved their intended effect:: + + # Capture baseline profile + python -m profiling.sampling run --binary -o baseline.bin script.py + + # After modifying code, generate differential flamegraph + python -m profiling.sampling run --diff-flamegraph baseline.bin -o diff.html script.py + +The visualization draws the current profile with frame widths showing current +time consumption, then applies color to indicate how each function changed +relative to the baseline. + +**Color coding**: + +- **Red**: Functions consuming more time (regressions). Lighter shades indicate + modest increases, while darker shades show severe regressions. + +- **Blue**: Functions consuming less time (improvements). Lighter shades for + modest reductions, darker shades for significant speedups. + +- **Gray**: Minimal or no change. + +- **Purple**: New functions not present in the baseline. + +Frame colors indicate changes in **direct time** (time when the function was at +the top of the stack, actively executing), not cumulative time including callees. +Hovering over a frame shows comparison details including baseline time, current +time, and the percentage change. + +Some call paths may disappear entirely between profiles. These are called +**elided stacks** and occur when optimizations eliminate code paths or certain +branches stop executing. If elided stacks are present, an elided toggle appears +allowing you to switch between the main differential view and an elided-only +view that shows just the removed paths (colored purple). + + Gecko format ------------ @@ -1488,6 +1529,10 @@ Output options Generate self-contained HTML flame graph. +.. option:: --diff-flamegraph + + Generate differential flamegraph comparing to a baseline binary profile. + .. option:: --gecko Generate Gecko JSON format for Firefox Profiler. diff --git a/Doc/library/select.rst b/Doc/library/select.rst index f6d8ce3c30ff1d..6c4a55612180a3 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -62,7 +62,7 @@ The module defines the following: *sizehint* informs epoll about the expected number of events to be registered. It must be positive, or ``-1`` to use the default. It is only - used on older systems where :c:func:`!epoll_create1` is not available; + used on older systems where :manpage:`epoll_create1(2)` is not available; otherwise it has no effect (though its value is still checked). *flags* is deprecated and completely ignored. However, when supplied, its @@ -89,6 +89,11 @@ The module defines the following: The *flags* parameter. ``select.EPOLL_CLOEXEC`` is used by default now. Use :func:`os.set_inheritable` to make the file descriptor inheritable. + .. versionchanged:: next + + When CPython is built, this function may be disabled using + :option:`--disable-epoll`. + .. function:: poll() diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index c3fe9943ba9d76..12ad45f557e6db 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -230,6 +230,8 @@ The variables defined in the :mod:`!signal` module are: Stop executing (cannot be caught or ignored). + .. availability:: Unix. + .. data:: SIGSTKFLT Stack fault on coprocessor. The Linux kernel does not raise this signal: it diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 24ce0ee56d92a6..71747d5f515a06 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -39,6 +39,8 @@ is implicit on send operations. A TLS/SSL wrapper for socket objects. +.. _socket-addresses: + Socket families --------------- @@ -903,7 +905,7 @@ The following functions all create :ref:`socket objects `. Build a pair of connected socket objects using the given address family, socket type, and protocol number. Address family, socket type, and protocol number are - as for the :func:`~socket.socket` function above. The default family is :const:`AF_UNIX` + as for the :func:`~socket.socket` function. The default family is :const:`AF_UNIX` if defined on the platform; otherwise, the default is :const:`AF_INET`. The newly created sockets are :ref:`non-inheritable `. @@ -999,8 +1001,8 @@ The following functions all create :ref:`socket objects `. Duplicate the file descriptor *fd* (an integer as returned by a file object's :meth:`~io.IOBase.fileno` method) and build a socket object from the result. Address - family, socket type and protocol number are as for the :func:`~socket.socket` function - above. The file descriptor should refer to a socket, but this is not checked --- + family, socket type and protocol number are as for the :func:`~socket.socket` function. + The file descriptor should refer to a socket, but this is not checked --- subsequent operations on the object may fail if the file descriptor is invalid. This function is rarely needed, but can be used to get or set socket options on a socket passed to a program as standard input or output (such as a server @@ -1564,8 +1566,8 @@ to sockets. .. method:: socket.bind(address) - Bind the socket to *address*. The socket must not already be bound. (The format - of *address* depends on the address family --- see above.) + Bind the socket to *address*. The socket must not already be bound. The format + of *address* depends on the address family --- see :ref:`socket-addresses`. .. audit-event:: socket.bind self,address socket.socket.bind @@ -1598,8 +1600,8 @@ to sockets. .. method:: socket.connect(address) - Connect to a remote socket at *address*. (The format of *address* depends on the - address family --- see above.) + Connect to a remote socket at *address*. The format of *address* depends on the + address family --- see :ref:`socket-addresses`. If the connection is interrupted by a signal, the method waits until the connection completes, or raises a :exc:`TimeoutError` on timeout, if the @@ -1674,16 +1676,16 @@ to sockets. .. method:: socket.getpeername() Return the remote address to which the socket is connected. This is useful to - find out the port number of a remote IPv4/v6 socket, for instance. (The format - of the address returned depends on the address family --- see above.) On some - systems this function is not supported. + find out the port number of a remote IPv4/v6 socket, for instance. The format + of the address returned depends on the address family --- see :ref:`socket-addresses`. + On some systems this function is not supported. .. method:: socket.getsockname() Return the socket's own address. This is useful to find out the port number of - an IPv4/v6 socket, for instance. (The format of the address returned depends on - the address family --- see above.) + an IPv4/v6 socket, for instance. The format of the address returned depends on + the address family --- see :ref:`socket-addresses`. .. method:: socket.getsockopt(level, optname[, buflen]) @@ -1795,7 +1797,8 @@ to sockets. where *bytes* is a bytes object representing the data received and *address* is the address of the socket sending the data. See the Unix manual page :manpage:`recv(2)` for the meaning of the optional argument *flags*; it defaults - to zero. (The format of *address* depends on the address family --- see above.) + to zero. The format of *address* depends on the address family --- see + :ref:`socket-addresses`. .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise @@ -1925,8 +1928,8 @@ to sockets. new bytestring. The return value is a pair ``(nbytes, address)`` where *nbytes* is the number of bytes received and *address* is the address of the socket sending the data. See the Unix manual page :manpage:`recv(2)` for the meaning of the - optional argument *flags*; it defaults to zero. (The format of *address* - depends on the address family --- see above.) + optional argument *flags*; it defaults to zero. The format of *address* + depends on the address family --- see :ref:`socket-addresses`. .. method:: socket.recv_into(buffer[, nbytes[, flags]]) @@ -1941,7 +1944,7 @@ to sockets. .. method:: socket.send(bytes[, flags]) Send data to the socket. The socket must be connected to a remote socket. The - optional *flags* argument has the same meaning as for :meth:`recv` above. + optional *flags* argument has the same meaning as for :meth:`recv`. Returns the number of bytes sent. Applications are responsible for checking that all data has been sent; if only some of the data was transmitted, the application needs to attempt delivery of the remaining data. For further @@ -1956,7 +1959,7 @@ to sockets. .. method:: socket.sendall(bytes[, flags]) Send data to the socket. The socket must be connected to a remote socket. The - optional *flags* argument has the same meaning as for :meth:`recv` above. + optional *flags* argument has the same meaning as for :meth:`recv`. Unlike :meth:`send`, this method continues to send data from *bytes* until either all data has been sent or an error occurs. ``None`` is returned on success. On error, an exception is raised, and there is no way to determine how @@ -1977,9 +1980,9 @@ to sockets. Send data to the socket. The socket should not be connected to a remote socket, since the destination socket is specified by *address*. The optional *flags* - argument has the same meaning as for :meth:`recv` above. Return the number of - bytes sent. (The format of *address* depends on the address family --- see - above.) + argument has the same meaning as for :meth:`recv`. Return the number of + bytes sent. The format of *address* depends on the address family --- see + :ref:`socket-addresses`. .. audit-event:: socket.sendto self,address socket.socket.sendto diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 40d103c13f8f38..ff1676c5bfb561 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -2285,7 +2285,7 @@ This section shows recipes for common adapters and converters. .. testcode:: - import datetime + import datetime as dt import sqlite3 def adapt_date_iso(val): @@ -2300,21 +2300,21 @@ This section shows recipes for common adapters and converters. """Adapt datetime.datetime to Unix timestamp.""" return int(val.timestamp()) - sqlite3.register_adapter(datetime.date, adapt_date_iso) - sqlite3.register_adapter(datetime.datetime, adapt_datetime_iso) - sqlite3.register_adapter(datetime.datetime, adapt_datetime_epoch) + sqlite3.register_adapter(dt.date, adapt_date_iso) + sqlite3.register_adapter(dt.datetime, adapt_datetime_iso) + sqlite3.register_adapter(dt.datetime, adapt_datetime_epoch) def convert_date(val): """Convert ISO 8601 date to datetime.date object.""" - return datetime.date.fromisoformat(val.decode()) + return dt.date.fromisoformat(val.decode()) def convert_datetime(val): """Convert ISO 8601 datetime to datetime.datetime object.""" - return datetime.datetime.fromisoformat(val.decode()) + return dt.datetime.fromisoformat(val.decode()) def convert_timestamp(val): """Convert Unix epoch timestamp to datetime.datetime object.""" - return datetime.datetime.fromtimestamp(int(val)) + return dt.datetime.fromtimestamp(int(val)) sqlite3.register_converter("date", convert_date) sqlite3.register_converter("datetime", convert_datetime) @@ -2323,17 +2323,17 @@ This section shows recipes for common adapters and converters. .. testcode:: :hide: - dt = datetime.datetime(2019, 5, 18, 15, 17, 8, 123456) + when = dt.datetime(2019, 5, 18, 15, 17, 8, 123456) - assert adapt_date_iso(dt.date()) == "2019-05-18" - assert convert_date(b"2019-05-18") == dt.date() + assert adapt_date_iso(when.date()) == "2019-05-18" + assert convert_date(b"2019-05-18") == when.date() - assert adapt_datetime_iso(dt) == "2019-05-18T15:17:08.123456" - assert convert_datetime(b"2019-05-18T15:17:08.123456") == dt + assert adapt_datetime_iso(when) == "2019-05-18T15:17:08.123456" + assert convert_datetime(b"2019-05-18T15:17:08.123456") == when # Using current time as fromtimestamp() returns local date/time. # Dropping microseconds as adapt_datetime_epoch truncates fractional second part. - now = datetime.datetime.now().replace(microsecond=0) + now = dt.datetime.now().replace(microsecond=0) current_timestamp = int(now.timestamp()) assert adapt_datetime_epoch(now) == current_timestamp diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index e83c2c9a8bc792..f2c35d1897a77f 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -67,7 +67,7 @@ by SSL sockets created through the :meth:`SSLContext.wrap_socket` method. Use of deprecated constants and functions result in deprecation warnings. -Functions, Constants, and Exceptions +Functions, constants, and exceptions ------------------------------------ @@ -374,7 +374,7 @@ Certificate handling .. function:: cert_time_to_seconds(cert_time) - Return the time in seconds since the Epoch, given the ``cert_time`` + Return the time in seconds since the epoch, given the ``cert_time`` string representing the "notBefore" or "notAfter" date from a certificate in ``"%b %d %H:%M:%S %Y %Z"`` strptime format (C locale). @@ -384,12 +384,12 @@ Certificate handling .. doctest:: newcontext >>> import ssl + >>> import datetime as dt >>> timestamp = ssl.cert_time_to_seconds("Jan 5 09:34:43 2018 GMT") >>> timestamp # doctest: +SKIP 1515144883 - >>> from datetime import datetime - >>> print(datetime.utcfromtimestamp(timestamp)) # doctest: +SKIP - 2018-01-05 09:34:43 + >>> print(dt.datetime.fromtimestamp(timestamp, dt.UTC)) # doctest: +SKIP + 2018-01-05 09:34:43+00:00 "notBefore" or "notAfter" dates must use GMT (:rfc:`5280`). @@ -1072,7 +1072,7 @@ Constants :attr:`TLSVersion.TLSv1_3` are deprecated. -SSL Sockets +SSL sockets ----------- .. class:: SSLSocket(socket.socket) @@ -1462,7 +1462,7 @@ SSL sockets also have the following additional methods and attributes: .. versionadded:: 3.6 -SSL Contexts +SSL contexts ------------ .. versionadded:: 3.2 @@ -2653,7 +2653,7 @@ thus several things you need to be aware of: as well. -Memory BIO Support +Memory BIO support ------------------ .. versionadded:: 3.5 diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 48291622d7be89..2099ef56169ede 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1163,13 +1163,13 @@ Sequence types also support the following methods: Return the total number of occurrences of *value* in *sequence*. -.. method:: list.index(value[, start[, stop]) - range.index(value[, start[, stop]) - tuple.index(value[, start[, stop]) +.. method:: list.index(value[, start[, stop]]) + range.index(value[, start[, stop]]) + tuple.index(value[, start[, stop]]) :no-contents-entry: :no-index-entry: :no-typesetting: -.. method:: sequence.index(value[, start[, stop]) +.. method:: sequence.index(value[, start[, stop]]) Return the index of the first occurrence of *value* in *sequence*. @@ -3735,12 +3735,13 @@ arbitrary binary data. The separator to search for may be any :term:`bytes-like object`. -.. method:: bytes.replace(old, new, count=-1, /) - bytearray.replace(old, new, count=-1, /) +.. method:: bytes.replace(old, new, /, count=-1) + bytearray.replace(old, new, /, count=-1) Return a copy of the sequence with all occurrences of subsequence *old* - replaced by *new*. If the optional argument *count* is given, only the - first *count* occurrences are replaced. + replaced by *new*. If *count* is given, only the first *count* occurrences + are replaced. If *count* is not specified or ``-1``, then all occurrences + are replaced. The subsequence to search for and its replacement may be any :term:`bytes-like object`. @@ -3750,6 +3751,9 @@ arbitrary binary data. The bytearray version of this method does *not* operate in place - it always produces a new object, even if no changes were made. + .. versionchanged:: next + *count* is now supported as a keyword argument. + .. method:: bytes.rfind(sub[, start[, end]]) bytearray.rfind(sub[, start[, end]]) diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 8096d90317d93f..08ccdfa3f454f8 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -82,7 +82,7 @@ The constants defined in this module are: .. _string-formatting: -Custom String Formatting +Custom string formatting ------------------------ The built-in string class provides the ability to do complex variable @@ -192,7 +192,7 @@ implementation as the built-in :meth:`~str.format` method. .. _formatstrings: -Format String Syntax +Format string syntax -------------------- The :meth:`str.format` method and the :class:`Formatter` class share the same @@ -304,7 +304,7 @@ See the :ref:`formatexamples` section for some examples. .. _formatspec: -Format Specification Mini-Language +Format specification mini-language ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ "Format specifications" are used within replacement fields contained within a @@ -759,8 +759,8 @@ Expressing a percentage:: Using type-specific formatting:: - >>> import datetime - >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58) + >>> import datetime as dt + >>> d = dt.datetime(2010, 7, 4, 12, 15, 58) >>> '{:%Y-%m-%d %H:%M:%S}'.format(d) '2010-07-04 12:15:58' diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index c08df5341282e7..fa0fb19d852f86 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -254,7 +254,7 @@ platform-dependent. +--------+--------------------------+--------------------+----------------+------------+ | ``N`` | :c:type:`size_t` | integer | | \(3) | +--------+--------------------------+--------------------+----------------+------------+ -| ``e`` | \(6) | float | 2 | \(4) | +| ``e`` | :c:expr:`_Float16` | float | 2 | \(4), \(6) | +--------+--------------------------+--------------------+----------------+------------+ | ``f`` | :c:expr:`float` | float | 4 | \(4) | +--------+--------------------------+--------------------+----------------+------------+ @@ -280,6 +280,12 @@ platform-dependent. .. versionchanged:: 3.14 Added support for the ``'F'`` and ``'D'`` formats. +.. seealso:: + + The :mod:`array` and :ref:`ctypes ` modules, + as well as third-party modules like `numpy `__, + use similar -- but slightly different -- type codes. + Notes: @@ -322,7 +328,9 @@ Notes: revision of the `IEEE 754 standard `_. It has a sign bit, a 5-bit exponent and 11-bit precision (with 10 bits explicitly stored), and can represent numbers between approximately ``6.1e-05`` and ``6.5e+04`` - at full precision. This type is not widely supported by C compilers: on a + at full precision. This type is not widely supported by C compilers: + it's available as :c:expr:`_Float16` type, if the compiler supports the Annex H + of the C23 standard. On a typical machine, an unsigned short can be used for storage, but not for math operations. See the Wikipedia page on the `half-precision floating-point format `_ for more information. diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index def6d58eabbeee..fe64daa3291d67 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -627,6 +627,12 @@ functions. the value in ``pw_uid`` will be used. If the value is an integer, it will be passed verbatim. (POSIX only) + .. note:: + + Specifying *user* will not drop existing supplementary group memberships! + The caller must also pass ``extra_groups=()`` to reduce the group membership + of the child process for security purposes. + .. availability:: POSIX .. versionadded:: 3.9 @@ -964,6 +970,11 @@ Reassigning them to new values is unsupported: A negative value ``-N`` indicates that the child was terminated by signal ``N`` (POSIX only). + When ``shell=True``, the return code reflects the exit status of the shell + itself (e.g. ``/bin/sh``), which may map signals to codes such as + ``128+N``. See the documentation of the shell (for example, the Bash + manual's Exit Status) for details. + Windows Popen Helpers --------------------- diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index bc12061a2aeb2d..aed7e7556f6666 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -143,21 +143,24 @@ The module defines three convenience functions and a public class: timeit.Timer('for i in range(10): oct(i)', 'gc.enable()').timeit() - .. method:: Timer.autorange(callback=None) + .. method:: Timer.autorange(callback=None, target_time=None) Automatically determine how many times to call :meth:`.timeit`. This is a convenience function that calls :meth:`.timeit` repeatedly - so that the total time >= 0.2 second, returning the eventual + so that the total time >= *Timer.target_time* seconds, returning the eventual (number of loops, time taken for that number of loops). It calls :meth:`.timeit` with increasing numbers from the sequence 1, 2, 5, - 10, 20, 50, ... until the time taken is at least 0.2 seconds. + 10, 20, 50, ... until the time taken is at least *target_time* seconds. If *callback* is given and is not ``None``, it will be called after each trial with two arguments: ``callback(number, time_taken)``. .. versionadded:: 3.6 + .. versionchanged:: next + The optional *target_time* parameter was added. + .. method:: Timer.repeat(repeat=5, number=1000000) @@ -239,6 +242,13 @@ Where the following options are understood: .. versionadded:: 3.5 +.. option:: -t, --target-time=T + + if :option:`--number` is 0, the code will run until it takes at + least this many seconds (default: 0.2) + + .. versionadded:: next + .. option:: -v, --verbose print raw timing results; repeat for more digits precision @@ -254,7 +264,7 @@ similarly. If :option:`-n` is not given, a suitable number of loops is calculated by trying increasing numbers from the sequence 1, 2, 5, 10, 20, 50, ... until the total -time is at least 0.2 seconds. +time is at least :option:`--target-time` seconds (default: 0.2). :func:`default_timer` measurements can be affected by other programs running on the same machine, so the best thing to do when accurate timing is necessary is diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index 7c81f52165972a..b8aa04b9ab246d 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -25,7 +25,7 @@ Using Mock ---------- -Mock Patching Methods +Mock patching methods ~~~~~~~~~~~~~~~~~~~~~ Common uses for :class:`Mock` objects include: @@ -71,7 +71,7 @@ the ``something`` method: -Mock for Method Calls on an Object +Mock for method calls on an object ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In the last example we patched a method directly on an object to check that it @@ -101,7 +101,7 @@ accessing it in the test will create it, but :meth:`~Mock.assert_called_with` will raise a failure exception. -Mocking Classes +Mocking classes ~~~~~~~~~~~~~~~ A common use case is to mock out classes instantiated by your code under test. @@ -139,7 +139,7 @@ name is also propagated to attributes or methods of the mock: -Tracking all Calls +Tracking all calls ~~~~~~~~~~~~~~~~~~ Often you want to track more than a single call to a method. The @@ -176,7 +176,7 @@ possible to track nested calls where the parameters used to create ancestors are True -Setting Return Values and Attributes +Setting return values and attributes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Setting the return values on a mock object is trivially easy: @@ -317,7 +317,7 @@ return an async function. >>> mock_instance.__aexit__.assert_awaited_once() -Creating a Mock from an Existing Object +Creating a mock from an existing object ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ One problem with over use of mocking is that it couples your tests to the @@ -384,7 +384,7 @@ contents per file stored in a dictionary:: assert file2.read() == "default" -Patch Decorators +Patch decorators ---------------- .. note:: @@ -518,7 +518,7 @@ decorator individually to every method whose name starts with "test". .. _further-examples: -Further Examples +Further examples ---------------- @@ -614,13 +614,13 @@ attribute on the mock date class is then set to a lambda function that returns a real date. When the mock date class is called a real date will be constructed and returned by ``side_effect``. :: - >>> from datetime import date + >>> import datetime as dt >>> with patch('mymodule.date') as mock_date: - ... mock_date.today.return_value = date(2010, 10, 8) - ... mock_date.side_effect = lambda *args, **kw: date(*args, **kw) + ... mock_date.today.return_value = dt.date(2010, 10, 8) + ... mock_date.side_effect = lambda *args, **kw: dt.date(*args, **kw) ... - ... assert mymodule.date.today() == date(2010, 10, 8) - ... assert mymodule.date(2009, 6, 8) == date(2009, 6, 8) + ... assert mymodule.date.today() == dt.date(2010, 10, 8) + ... assert mymodule.date(2009, 6, 8) == dt.date(2009, 6, 8) Note that we don't patch :class:`datetime.date` globally, we patch ``date`` in the module that *uses* it. See :ref:`where to patch `. @@ -638,7 +638,7 @@ is discussed in `this blog entry `_. -Mocking a Generator Method +Mocking a generator method ~~~~~~~~~~~~~~~~~~~~~~~~~~ A Python generator is a function or method that uses the :keyword:`yield` statement @@ -739,7 +739,7 @@ exception is raised in the setUp then tearDown is not called. >>> MyTest('test_foo').run() -Mocking Unbound Methods +Mocking unbound methods ~~~~~~~~~~~~~~~~~~~~~~~ Sometimes a test needs to patch an *unbound method*, which means patching the @@ -937,7 +937,7 @@ and the ``return_value`` will use your subclass automatically. That means all children of a ``CopyingMock`` will also have the type ``CopyingMock``. -Nesting Patches +Nesting patches ~~~~~~~~~~~~~~~ Using patch as a context manager is nice, but if you do multiple patches you diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 919d4c595bf793..bbb15ce5e758c6 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -691,7 +691,7 @@ Functions .. versionadded:: 3.2 -.. function:: SubElement(parent, tag, attrib={}, **extra) +.. function:: SubElement(parent, tag, /, attrib={}, **extra) Subelement factory. This function creates an element instance, and appends it to an existing element. @@ -705,6 +705,9 @@ Functions .. versionchanged:: 3.15 *attrib* can now be a :class:`frozendict`. + .. versionchanged:: next + *parent* and *tag* are now positional-only parameters. + .. function:: tostring(element, encoding="us-ascii", method="xml", *, \ xml_declaration=None, default_namespace=None, \ @@ -880,7 +883,7 @@ Element Objects :noindex: :no-index: -.. class:: Element(tag, attrib={}, **extra) +.. class:: Element(tag, /, attrib={}, **extra) Element class. This class defines the Element interface, and provides a reference implementation of this interface. @@ -893,6 +896,9 @@ Element Objects .. versionchanged:: 3.15 *attrib* can now be a :class:`frozendict`. + .. versionchanged:: next + *tag* is now a positional-only parameter. + .. attribute:: tag diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index 8b3b3dcaa13447..458e67dbe51dcc 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -272,12 +272,12 @@ DateTime Objects A working example follows. The server code:: - import datetime + import datetime as dt from xmlrpc.server import SimpleXMLRPCServer import xmlrpc.client def today(): - today = datetime.datetime.today() + today = dt.datetime.today() return xmlrpc.client.DateTime(today) server = SimpleXMLRPCServer(("localhost", 8000)) @@ -288,14 +288,14 @@ A working example follows. The server code:: The client code for the preceding server:: import xmlrpc.client - import datetime + import datetime as dt proxy = xmlrpc.client.ServerProxy("http://localhost:8000/") today = proxy.today() - # convert the ISO8601 string to a datetime object - converted = datetime.datetime.strptime(today.value, "%Y%m%dT%H:%M:%S") - print("Today: %s" % converted.strftime("%d.%m.%Y, %H:%M")) + # convert the ISO 8601 string to a datetime object + converted = dt.datetime.strptime(today.value, "%Y%m%dT%H:%M:%S") + print(f"Today: {converted.strftime('%d.%m.%Y, %H:%M')}") .. _binary-objects: diff --git a/Doc/library/xmlrpc.server.rst b/Doc/library/xmlrpc.server.rst index 2c130785be0db0..5702257dfe2eba 100644 --- a/Doc/library/xmlrpc.server.rst +++ b/Doc/library/xmlrpc.server.rst @@ -69,7 +69,7 @@ servers written in Python. Servers can either be free standing, using .. _simple-xmlrpc-servers: -SimpleXMLRPCServer Objects +SimpleXMLRPCServer objects -------------------------- The :class:`SimpleXMLRPCServer` class is based on @@ -140,7 +140,7 @@ alone XML-RPC servers. .. _simplexmlrpcserver-example: -SimpleXMLRPCServer Example +SimpleXMLRPCServer example ^^^^^^^^^^^^^^^^^^^^^^^^^^ Server code:: @@ -231,7 +231,7 @@ a server allowing dotted names and registering a multicall function. :: - import datetime + import datetime as dt class ExampleService: def getData(self): @@ -240,7 +240,7 @@ a server allowing dotted names and registering a multicall function. class currentTime: @staticmethod def getCurrentTime(): - return datetime.datetime.now() + return dt.datetime.now() with SimpleXMLRPCServer(("localhost", 8000)) as server: server.register_function(pow) @@ -387,7 +387,7 @@ to HTTP GET requests. Servers can either be free standing, using .. _doc-xmlrpc-servers: -DocXMLRPCServer Objects +DocXMLRPCServer objects ----------------------- The :class:`DocXMLRPCServer` class is derived from :class:`SimpleXMLRPCServer` diff --git a/Doc/library/zoneinfo.rst b/Doc/library/zoneinfo.rst index cba08d6614fc08..f5d3ade478ffb2 100644 --- a/Doc/library/zoneinfo.rst +++ b/Doc/library/zoneinfo.rst @@ -37,24 +37,24 @@ the constructor, the :meth:`datetime.replace ` method or :meth:`datetime.astimezone `:: >>> from zoneinfo import ZoneInfo - >>> from datetime import datetime, timedelta + >>> import datetime as dt - >>> dt = datetime(2020, 10, 31, 12, tzinfo=ZoneInfo("America/Los_Angeles")) - >>> print(dt) + >>> when = dt.datetime(2020, 10, 31, 12, tzinfo=ZoneInfo("America/Los_Angeles")) + >>> print(when) 2020-10-31 12:00:00-07:00 - >>> dt.tzname() + >>> when.tzname() 'PDT' Datetimes constructed in this way are compatible with datetime arithmetic and handle daylight saving time transitions with no further intervention:: - >>> dt_add = dt + timedelta(days=1) + >>> when_add = when + dt.timedelta(days=1) - >>> print(dt_add) + >>> print(when_add) 2020-11-01 12:00:00-08:00 - >>> dt_add.tzname() + >>> when_add.tzname() 'PST' These time zones also support the :attr:`~datetime.datetime.fold` attribute @@ -63,26 +63,25 @@ times (such as a daylight saving time to standard time transition), the offset from *before* the transition is used when ``fold=0``, and the offset *after* the transition is used when ``fold=1``, for example:: - >>> dt = datetime(2020, 11, 1, 1, tzinfo=ZoneInfo("America/Los_Angeles")) - >>> print(dt) + >>> when = dt.datetime(2020, 11, 1, 1, tzinfo=ZoneInfo("America/Los_Angeles")) + >>> print(when) 2020-11-01 01:00:00-07:00 - >>> print(dt.replace(fold=1)) + >>> print(when.replace(fold=1)) 2020-11-01 01:00:00-08:00 When converting from another time zone, the fold will be set to the correct value:: - >>> from datetime import timezone >>> LOS_ANGELES = ZoneInfo("America/Los_Angeles") - >>> dt_utc = datetime(2020, 11, 1, 8, tzinfo=timezone.utc) + >>> when_utc = dt.datetime(2020, 11, 1, 8, tzinfo=dt.timezone.utc) >>> # Before the PDT -> PST transition - >>> print(dt_utc.astimezone(LOS_ANGELES)) + >>> print(when_utc.astimezone(LOS_ANGELES)) 2020-11-01 01:00:00-07:00 >>> # After the PDT -> PST transition - >>> print((dt_utc + timedelta(hours=1)).astimezone(LOS_ANGELES)) + >>> print((when_utc + dt.timedelta(hours=1)).astimezone(LOS_ANGELES)) 2020-11-01 01:00:00-08:00 Data sources @@ -276,8 +275,8 @@ the note on usage in the attribute documentation):: >>> str(zone) 'Pacific/Kwajalein' - >>> dt = datetime(2020, 4, 1, 3, 15, tzinfo=zone) - >>> f"{dt.isoformat()} [{dt.tzinfo}]" + >>> when = dt.datetime(2020, 4, 1, 3, 15, tzinfo=zone) + >>> f"{when.isoformat()} [{when.tzinfo}]" '2020-04-01T03:15:00+12:00 [Pacific/Kwajalein]' For objects constructed from a file without specifying a ``key`` parameter, diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index ae541680c534d6..f3ed1539493b6a 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -560,7 +560,7 @@ start with a character in the "letter-like" set ``xid_start``, and the remaining characters must be in the "letter- and digit-like" set ``xid_continue``. -These sets based on the *XID_Start* and *XID_Continue* sets as defined by the +These sets are based on the *XID_Start* and *XID_Continue* sets as defined by the Unicode standard annex `UAX-31`_. Python's ``xid_start`` additionally includes the underscore (``_``). Note that Python does not necessarily conform to `UAX-31`_. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index ffe98d332d6e93..189173a5f8a75f 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -2,7 +2,6 @@ # as tested on the CI via check-warnings.py in reusable-docs.yml. # Keep lines sorted lexicographically to help avoid merge conflicts. -Doc/c-api/descriptor.rst Doc/c-api/init_config.rst Doc/c-api/intro.rst Doc/c-api/stable.rst diff --git a/Doc/tools/check-html-ids.py b/Doc/tools/check-html-ids.py index 8e8e0a581df72d..7d86c6cc3264ad 100644 --- a/Doc/tools/check-html-ids.py +++ b/Doc/tools/check-html-ids.py @@ -175,6 +175,7 @@ def verbose_print(*args, **kwargs): ) if args.exclude_file: print(f'Alternatively, add them to {args.exclude_file}.') + sys.exit(1) if __name__ == '__main__': diff --git a/Doc/tools/removed-ids.txt b/Doc/tools/removed-ids.txt new file mode 100644 index 00000000000000..f3cd8bf0ef5bb9 --- /dev/null +++ b/Doc/tools/removed-ids.txt @@ -0,0 +1 @@ +# HTML IDs excluded from the check-html-ids.py check. diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index d3541c604683e4..276e31a3056f0e 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -15,20 +15,20 @@ More on Lists The :ref:`list ` data type has some more methods. Here are all of the methods of list objects: -.. method:: list.append(x) +.. method:: list.append(value, /) :noindex: Add an item to the end of the list. Similar to ``a[len(a):] = [x]``. -.. method:: list.extend(iterable) +.. method:: list.extend(iterable, /) :noindex: Extend the list by appending all the items from the iterable. Similar to ``a[len(a):] = iterable``. -.. method:: list.insert(i, x) +.. method:: list.insert(index, value, /) :noindex: Insert an item at a given position. The first argument is the index of the @@ -36,14 +36,14 @@ of the methods of list objects: the list, and ``a.insert(len(a), x)`` is equivalent to ``a.append(x)``. -.. method:: list.remove(x) +.. method:: list.remove(value, /) :noindex: - Remove the first item from the list whose value is equal to *x*. It raises a + Remove the first item from the list whose value is equal to *value*. It raises a :exc:`ValueError` if there is no such item. -.. method:: list.pop([i]) +.. method:: list.pop(index=-1, /) :noindex: Remove the item at the given position in the list, and return it. If no index @@ -58,10 +58,10 @@ of the methods of list objects: Remove all items from the list. Similar to ``del a[:]``. -.. method:: list.index(x[, start[, end]]) +.. method:: list.index(value[, start[, stop]]) :noindex: - Return zero-based index of the first occurrence of *x* in the list. + Return zero-based index of the first occurrence of *value* in the list. Raises a :exc:`ValueError` if there is no such item. The optional arguments *start* and *end* are interpreted as in the slice @@ -70,10 +70,10 @@ of the methods of list objects: sequence rather than the *start* argument. -.. method:: list.count(x) +.. method:: list.count(value, /) :noindex: - Return the number of times *x* appears in the list. + Return the number of times *value* appears in the list. .. method:: list.sort(*, key=None, reverse=False) diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 342c1a00193959..e7c64104474050 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -1,13 +1,13 @@ .. _tut-brieftour: ********************************** -Brief Tour of the Standard Library +Brief tour of the standard library ********************************** .. _tut-os-interface: -Operating System Interface +Operating system interface ========================== The :mod:`os` module provides dozens of functions for interacting with the @@ -47,7 +47,7 @@ a higher level interface that is easier to use:: .. _tut-file-wildcards: -File Wildcards +File wildcards ============== The :mod:`glob` module provides a function for making file lists from directory @@ -60,7 +60,7 @@ wildcard searches:: .. _tut-command-line-arguments: -Command Line Arguments +Command-line arguments ====================== Common utility scripts often need to process command line arguments. These @@ -97,7 +97,7 @@ to ``['alpha.txt', 'beta.txt']``. .. _tut-stderr: -Error Output Redirection and Program Termination +Error output redirection and program termination ================================================ The :mod:`sys` module also has attributes for *stdin*, *stdout*, and *stderr*. @@ -112,7 +112,7 @@ The most direct way to terminate a script is to use ``sys.exit()``. .. _tut-string-pattern-matching: -String Pattern Matching +String pattern matching ======================= The :mod:`re` module provides regular expression tools for advanced string @@ -175,7 +175,7 @@ computations. .. _tut-internet-access: -Internet Access +Internet access =============== There are a number of modules for accessing the internet and processing internet @@ -206,7 +206,7 @@ from URLs and :mod:`smtplib` for sending mail:: .. _tut-dates-and-times: -Dates and Times +Dates and times =============== The :mod:`datetime` module supplies classes for manipulating dates and times in @@ -216,15 +216,15 @@ formatting and manipulation. The module also supports objects that are timezone aware. :: >>> # dates are easily constructed and formatted - >>> from datetime import date - >>> now = date.today() + >>> import datetime as dt + >>> now = dt.date.today() >>> now datetime.date(2003, 12, 2) >>> now.strftime("%m-%d-%y. %d %b %Y is a %A on the %d day of %B.") '12-02-03. 02 Dec 2003 is a Tuesday on the 02 day of December.' >>> # dates support calendar arithmetic - >>> birthday = date(1964, 7, 31) + >>> birthday = dt.date(1964, 7, 31) >>> age = now - birthday >>> age.days 14368 @@ -232,7 +232,7 @@ aware. :: .. _tut-data-compression: -Data Compression +Data compression ================ Common data archiving and compression formats are directly supported by modules @@ -254,7 +254,7 @@ including: :mod:`zlib`, :mod:`gzip`, :mod:`bz2`, :mod:`lzma`, :mod:`zipfile` and .. _tut-performance-measurement: -Performance Measurement +Performance measurement ======================= Some Python users develop a deep interest in knowing the relative performance of @@ -278,7 +278,7 @@ larger blocks of code. .. _tut-quality-control: -Quality Control +Quality control =============== One approach for developing high quality software is to write tests for each @@ -324,7 +324,7 @@ file:: .. _tut-batteries-included: -Batteries Included +Batteries included ================== Python has a "batteries included" philosophy. This is best seen through the diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index ce6872f3c0fda3..d0355ce47a6504 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -1132,6 +1132,14 @@ conflict. and kill the process. Only enable this in environments where the huge-page pool is properly sized and fork-safety is not a concern. + On Windows you need a special privilege. See the + `Windows documentation for large pages + `_ + for details. Python will fail on startup if the required privilege + `SeLockMemoryPrivilege + `_ + is not held by the user. + .. versionadded:: 3.15 @@ -1330,6 +1338,13 @@ conflict. .. versionadded:: 3.13 +.. envvar:: PYTHON_BASIC_COMPLETER + + If this variable is set to any value, PyREPL will use :mod:`rlcompleter` to + implement tab completion, instead of the default one which uses colors. + + .. versionadded:: 3.15 + .. envvar:: PYTHON_HISTORY This environment variable can be used to set the location of a diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 6bef290d181fc9..cc6aafe80f810d 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -463,6 +463,17 @@ General Options ``pkg-config`` options. +.. option:: --disable-epoll + + Build without ``epoll``, meaning that :py:func:`select.epoll` will not be + present even if the system provides an + :manpage:`epoll_create ` function. + This may be used on systems where :manpage:`!epoll_create` or + :manpage:`epoll_create1 ` is available + but incompatible with Linux semantics. + + .. versionadded:: next + C compiler options ------------------ @@ -795,9 +806,18 @@ also be used to improve performance. Even when compiled with this option, huge pages are **not** used at runtime unless the :envvar:`PYTHON_PYMALLOC_HUGEPAGES` environment variable is set - to ``1``. This opt-in is required because huge pages carry risks on Linux: - if the huge-page pool is exhausted, page faults (including copy-on-write - faults after :func:`os.fork`) deliver ``SIGBUS`` and kill the process. + to ``1``. This opt-in is required because huge pages + + * carry risks on Linux: if the huge-page pool is exhausted, page faults + (including copy-on-write faults after :func:`os.fork`) deliver ``SIGBUS`` + and kill the process. + + * need a special privilege on Windows. See the `Windows documentation for large pages + `_ + for details. Python will fail on startup if the required privilege + `SeLockMemoryPrivilege + `_ + is not held by the user. The configure script checks that the platform supports ``MAP_HUGETLB`` and emits a warning if it is not available. @@ -886,9 +906,11 @@ See also the :ref:`Python Development Mode ` and the :option:`--with-trace-refs` configure option. .. versionchanged:: 3.8 - Release builds and debug builds are now ABI compatible: defining the + Release builds are now ABI compatible with debug builds: defining the ``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro (see the - :option:`--with-trace-refs` option). + :option:`--with-trace-refs` option). However, debug builds still expose + more symbols than release builds and code built against a debug build is not + necessarily compatible with a release build. Debug options diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index f43692b3dce9e8..43ab19037d2627 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -1698,8 +1698,8 @@ current local date. Once created, instances of the date/time classes are all immutable. There are a number of methods for producing formatted strings from objects:: - >>> import datetime - >>> now = datetime.datetime.now() + >>> import datetime as dt + >>> now = dt.datetime.now() >>> now.isoformat() '2002-12-30T21:27:03.994956' >>> now.ctime() # Only available on date, datetime @@ -1710,10 +1710,10 @@ number of methods for producing formatted strings from objects:: The :meth:`~datetime.datetime.replace` method allows modifying one or more fields of a :class:`~datetime.date` or :class:`~datetime.datetime` instance, returning a new instance:: - >>> d = datetime.datetime.now() + >>> d = dt.datetime.now() >>> d datetime.datetime(2002, 12, 30, 22, 15, 38, 827738) - >>> d.replace(year=2001, hour = 12) + >>> d.replace(year=2001, hour=12) datetime.datetime(2001, 12, 30, 12, 15, 38, 827738) >>> diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index 9b8f36862c1335..03e612fb651e71 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -1313,10 +1313,10 @@ complete list of changes, or look through the SVN logs for all the details. by Josh Spoerri. It uses the same format characters as :func:`time.strptime` and :func:`time.strftime`:: - from datetime import datetime + import datetime as dt - ts = datetime.strptime('10:13:15 2006-03-07', - '%H:%M:%S %Y-%m-%d') + ts = dt.datetime.strptime('10:13:15 2006-03-07', + '%H:%M:%S %Y-%m-%d') * The :meth:`SequenceMatcher.get_matching_blocks` method in the :mod:`difflib` module now guarantees to return a minimal list of blocks describing matching diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index f5e3a47037c65f..1215601a09d681 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -2822,10 +2822,10 @@ Using the module is simple:: import sys import plistlib - import datetime + import datetime as dt # Create data structure - data_struct = dict(lastAccessed=datetime.datetime.now(), + data_struct = dict(lastAccessed=dt.datetime.now(), version=1, categories=('Personal','Shared','Private')) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 772334f40a56fb..dfdfe66be7e6cc 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -454,7 +454,7 @@ on x86-64 and AArch64 architectures. However, a future release of GCC is expected to support this as well. This feature is opt-in for now. Enabling profile-guided optimization is highly -recommendeded when using the new interpreter as it is the only configuration +recommended when using the new interpreter as it is the only configuration that has been tested and validated for improved performance. For further information, see :option:`--with-tail-call-interp`. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 5e6265a45231db..54679c60fc9370 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -84,7 +84,8 @@ Summary -- Release highlights ` * :ref:`The JIT compiler has been significantly upgraded ` * :ref:`Improved error messages ` - +* :ref:`The official Windows 64-bit binaries now use the tail-calling interpreter + ` New features ============ @@ -216,7 +217,8 @@ For example:: The following standard library modules have been updated to accept :class:`!frozendict`: :mod:`copy`, :mod:`decimal`, :mod:`json`, :mod:`marshal`, -:mod:`pickle`, :mod:`pprint` and :mod:`xml.etree.ElementTree`. +:mod:`plistlib` (only for serialization), :mod:`pickle`, :mod:`pprint` and +:mod:`xml.etree.ElementTree`. :func:`eval` and :func:`exec` accept :class:`!frozendict` for *globals*, and :func:`type` and :meth:`str.maketrans` accept :class:`!frozendict` for *dict*. @@ -598,6 +600,14 @@ Other language changes making it a :term:`generic type`. (Contributed by James Hilton-Balfe in :gh:`128335`.) +* The class :class:`memoryview` now supports the :c:expr:`float complex` and + :c:expr:`double complex` C types: formatting characters ``'F'`` and ``'D'`` + respectively. + (Contributed by Sergey B Kirpichev in :gh:`146151`.) + +* Allow the *count* argument of :meth:`bytes.replace` to be a keyword. + (Contributed by Stan Ulbrych in :gh:`147856`.) + New modules =========== @@ -629,22 +639,53 @@ argparse (Contributed by Savannah Ostrowski in :gh:`142390`.) +array +----- + +* Support the :c:expr:`float complex` and :c:expr:`double complex` C types: + formatting characters ``'F'`` and ``'D'`` respectively. + (Contributed by Sergey B Kirpichev in :gh:`146151`.) + +* Support half-floats (16-bit IEEE 754 binary interchange format): formatting + character ``'e'``. + (Contributed by Sergey B Kirpichev in :gh:`146238`.) + + base64 ------ * Added the *pad* parameter in :func:`~base64.z85encode`. (Contributed by Hauke Dämpfling in :gh:`143103`.) -* Added the *wrapcol* parameter in :func:`~base64.b64encode`. - (Contributed by Serhiy Storchaka in :gh:`143214`.) +* Added the *padded* parameter in + :func:`~base64.b32encode`, :func:`~base64.b32decode`, + :func:`~base64.b32hexencode`, :func:`~base64.b32hexdecode`, + :func:`~base64.b64encode`, :func:`~base64.b64decode`, + :func:`~base64.urlsafe_b64encode`, and :func:`~base64.urlsafe_b64decode`. + (Contributed by Serhiy Storchaka in :gh:`73613`.) + +* Added the *wrapcol* parameter in :func:`~base64.b16encode`, + :func:`~base64.b32encode`, :func:`~base64.b32hexencode`, + :func:`~base64.b64encode`, :func:`~base64.b85encode`, and + :func:`~base64.z85encode`. + (Contributed by Serhiy Storchaka in :gh:`143214` and :gh:`146431`.) -* Added the *ignorechars* parameter in :func:`~base64.b64decode`. - (Contributed by Serhiy Storchaka in :gh:`144001`.) +* Added the *ignorechars* parameter in :func:`~base64.b16decode`, + :func:`~base64.b32decode`, :func:`~base64.b32hexdecode`, + :func:`~base64.b64decode`, :func:`~base64.b85decode`, and + :func:`~base64.z85decode`. + (Contributed by Serhiy Storchaka in :gh:`144001` and :gh:`146431`.) binascii -------- +* Added functions for Base32 encoding: + + - :func:`~binascii.b2a_base32` and :func:`~binascii.a2b_base32` + + (Contributed by James Seo in :gh:`146192`.) + * Added functions for Ascii85, Base85, and Z85 encoding: - :func:`~binascii.b2a_ascii85` and :func:`~binascii.a2b_ascii85` @@ -652,6 +693,11 @@ binascii (Contributed by James Seo and Serhiy Storchaka in :gh:`101178`.) +* Added the *padded* parameter in + :func:`~binascii.b2a_base32`, :func:`~binascii.a2b_base32`, + :func:`~binascii.b2a_base64`, and :func:`~binascii.a2b_base64`. + (Contributed by Serhiy Storchaka in :gh:`73613`.) + * Added the *wrapcol* parameter in :func:`~binascii.b2a_base64`. (Contributed by Serhiy Storchaka in :gh:`143214`.) @@ -659,8 +705,9 @@ binascii :func:`~binascii.a2b_base64`. (Contributed by Serhiy Storchaka in :gh:`145980`.) -* Added the *ignorechars* parameter in :func:`~binascii.a2b_base64`. - (Contributed by Serhiy Storchaka in :gh:`144001`.) +* Added the *ignorechars* parameter in :func:`~binascii.a2b_hex`, + :func:`~binascii.unhexlify`, and :func:`~binascii.a2b_base64`. + (Contributed by Serhiy Storchaka in :gh:`144001` and :gh:`146431`.) calendar @@ -684,26 +731,6 @@ collections (Contributed by Raymond Hettinger in :gh:`138682`.) -collections.abc ---------------- - -* :class:`collections.abc.ByteString` has been removed from - ``collections.abc.__all__``. :class:`!collections.abc.ByteString` has been - deprecated since Python 3.12, and is scheduled for removal in Python 3.17. - -* The following statements now cause ``DeprecationWarning``\ s to be emitted at - runtime: - - * ``from collections.abc import ByteString`` - * ``import collections.abc; collections.abc.ByteString``. - - ``DeprecationWarning``\ s were already emitted if - :class:`collections.abc.ByteString` was subclassed or used as the second - argument to :func:`isinstance` or :func:`issubclass`, but warnings were not - previously emitted if it was merely imported or accessed from the - :mod:`!collections.abc` module. - - concurrent.futures ------------------ @@ -802,6 +829,17 @@ inspect for :func:`~inspect.getdoc`. (Contributed by Serhiy Storchaka in :gh:`132686`.) +json +---- + +* Add the *array_hook* parameter to :func:`~json.load` and + :func:`~json.loads` functions: + allow a callback for JSON literal array types to customize Python lists in + the resulting decoded object. Passing combined :class:`frozendict` to + *object_pairs_hook* param and :class:`tuple` to ``array_hook`` will yield a + deeply nested immutable Python structure representing the JSON data. + (Contributed by Joao S. O. Bueno in :gh:`146440`) + locale ------ @@ -830,12 +868,12 @@ mimetypes * Add ``application/dicom`` MIME type for ``.dcm`` extension. (Contributed by Benedikt Johannes in :gh:`144217`.) +* Add ``application/efi``. (Contributed by Charlie Lin in :gh:`145720`.) * Add ``application/node`` MIME type for ``.cjs`` extension. (Contributed by John Franey in :gh:`140937`.) * Add ``application/toml``. (Contributed by Gil Forcada in :gh:`139959`.) * Add ``application/sql`` and ``application/vnd.sqlite3``. (Contributed by Charlie Lin in :gh:`145698`.) -* Add ``image/jxl``. (Contributed by Foolbar in :gh:`144213`.) * Add the following MIME types: - ``application/vnd.ms-cab-compressed`` for ``.cab`` extension @@ -844,6 +882,7 @@ mimetypes (Contributed by Charlie Lin in :gh:`145718`.) +* Add ``image/jxl``. (Contributed by Foolbar in :gh:`144213`.) * Rename ``application/x-texinfo`` to ``application/texinfo``. (Contributed by Charlie Lin in :gh:`140165`.) * Changed the MIME type for ``.ai`` files to ``application/pdf``. @@ -953,7 +992,7 @@ ssl --- * Indicate through :data:`ssl.HAS_PSK_TLS13` whether the :mod:`ssl` module - supports "External PSKs" in TLSv1.3, as described in RFC 9258. + supports "External PSKs" in TLSv1.3, as described in :rfc:`9258`. (Contributed by Will Childs-Klein in :gh:`133624`.) * Added new methods for managing groups used for SSL key agreement @@ -1063,6 +1102,11 @@ timeit :ref:`environment variables `. (Contributed by Yi Hong in :gh:`139374`.) +* Make the target time of :meth:`timeit.Timer.autorange` configurable + and add ``--target-time`` option to the command-line interface. + (Contributed by Alessandro Cucci and Miikka Koskinen in :gh:`140283`.) + + tkinter ------- @@ -1139,7 +1183,7 @@ tomllib types ------- +----- * Expose the write-through :func:`locals` proxy type as :data:`types.FrameLocalsProxyType`. @@ -1147,6 +1191,40 @@ types as described in :pep:`667`. +typing +------ + +.. _whatsnew315-typeform: + +* :pep:`747`: Add :data:`~typing.TypeForm`, a new special form for annotating + values that are themselves type expressions. + ``TypeForm[T]`` means "a type form object describing ``T`` (or a type + assignable to ``T``)". At runtime, ``TypeForm(x)`` simply returns ``x``, + which allows explicit annotation of type-form values without changing + behavior. + + This helps libraries that accept user-provided type expressions + (for example ``int``, ``str | None``, :class:`~typing.TypedDict` + classes, or ``list[int]``) expose precise signatures: + + .. code-block:: python + + from typing import Any, TypeForm + + def cast[T](typ: TypeForm[T], value: Any) -> T: ... + + (Contributed by Jelle Zijlstra in :gh:`145033`.) + +* Code like ``class ExtraTypeVars(P1[S], Protocol[T, T2]): ...`` now raises + a :exc:`TypeError`, because ``S`` is not listed in ``Protocol`` parameters. + (Contributed by Nikita Sobolev in :gh:`137191`.) + +* Code like ``class B2(A[T2], Protocol[T1, T2]): ...`` now correctly handles + type parameters order: it is ``(T1, T2)``, not ``(T2, T1)`` + as it was incorrectly inferred in runtime before. + (Contributed by Nikita Sobolev in :gh:`137191`.) + + unicodedata ----------- @@ -1249,18 +1327,6 @@ zlib Optimizations ============= -* Builds using Visual Studio 2026 (MSVC 18) may now use the new - :ref:`tail-calling interpreter `. - Results on Visual Studio 18.1.1 report between - `15-20% `__ - speedup on the geometric mean of pyperformance on Windows x86-64 over - the switch-case interpreter on an AMD Ryzen 7 5800X. We have - observed speedups ranging from 14% for large pure-Python libraries - to 40% for long-running small pure-Python scripts on Windows. - This was made possible by a new feature introduced in MSVC 18. - (Contributed by Chris Eibl, Ken Jin, and Brandt Bucher in :gh:`143068`. - Special thanks to the MSVC team including Hulon Jenkins.) - * ``mimalloc`` is now used as the default allocator for for raw memory allocations such as via :c:func:`PyMem_RawMalloc` for better performance on :term:`free-threaded builds `. @@ -1279,6 +1345,10 @@ base64 & binascii two orders of magnitude less memory. (Contributed by James Seo and Serhiy Storchaka in :gh:`101178`.) +* Implementation for Base32 has been rewritten in C. + Encoding and decoding is now two orders of magnitude faster. + (Contributed by James Seo in :gh:`146192`) + csv --- @@ -1294,11 +1364,11 @@ Upgraded JIT compiler Results from the `pyperformance `__ benchmark suite report -`5-6% `__ +`6-7% `__ geometric mean performance improvement for the JIT over the standard CPython interpreter built with all optimizations enabled on x86-64 Linux. On AArch64 macOS, the JIT has a -`8-9% `__ +`12-13% `__ speedup over the :ref:`tail calling interpreter ` with all optimizations enabled. The speedups for JIT builds versus no JIT builds range from roughly 15% slowdown to over @@ -1379,6 +1449,14 @@ Diego Russo in :gh:`140683` and :gh:`142305`.) Removed ======== +collections.abc +--------------- + +* :class:`collections.abc.ByteString` has been removed from + ``collections.abc.__all__``. :class:`!collections.abc.ByteString` has been + deprecated since Python 3.12, and is scheduled for removal in Python 3.17. + + ctypes ------ @@ -1455,26 +1533,9 @@ threading typing ------ -.. _whatsnew315-typeform: - -* :pep:`747`: Add :data:`~typing.TypeForm`, a new special form for annotating - values that are themselves type expressions. - ``TypeForm[T]`` means "a type form object describing ``T`` (or a type - assignable to ``T``)". At runtime, ``TypeForm(x)`` simply returns ``x``, - which allows explicit annotation of type-form values without changing - behavior. - - This helps libraries that accept user-provided type expressions - (for example ``int``, ``str | None``, :class:`~typing.TypedDict` - classes, or ``list[int]``) expose precise signatures: - - .. code-block:: python - - from typing import Any, TypeForm - - def cast[T](typ: TypeForm[T], value: Any) -> T: ... - - (Contributed by Jelle Zijlstra in :gh:`145033`.) +* :class:`typing.ByteString` has been removed from ``typing.__all__``. + :class:`!typing.ByteString` has been deprecated since Python 3.9, and is + scheduled for removal in Python 3.17. * The undocumented keyword argument syntax for creating :class:`~typing.NamedTuple` classes (for example, @@ -1488,33 +1549,6 @@ typing or ``TD = TypedDict("TD", {})`` instead. (Contributed by Bénédikt Tran in :gh:`133823`.) -* Code like ``class ExtraTypeVars(P1[S], Protocol[T, T2]): ...`` now raises - a :exc:`TypeError`, because ``S`` is not listed in ``Protocol`` parameters. - (Contributed by Nikita Sobolev in :gh:`137191`.) - -* Code like ``class B2(A[T2], Protocol[T1, T2]): ...`` now correctly handles - type parameters order: it is ``(T1, T2)``, not ``(T2, T1)`` - as it was incorrectly inferred in runtime before. - (Contributed by Nikita Sobolev in :gh:`137191`.) - -* :class:`typing.ByteString` has been removed from ``typing.__all__``. - :class:`!typing.ByteString` has been deprecated since Python 3.9, and is - scheduled for removal in Python 3.17. - -* The following statements now cause ``DeprecationWarning``\ s to be emitted at - runtime: - - * ``from typing import ByteString`` - * ``import typing; typing.ByteString``. - - ``DeprecationWarning``\ s were already emitted if :class:`typing.ByteString` - was subclassed or used as the second argument to :func:`isinstance` or - :func:`issubclass`, but warnings were not previously emitted if it was merely - imported or accessed from the :mod:`!typing` module. - -* Deprecated :func:`!typing.no_type_check_decorator` has been removed. - (Contributed by Nikita Sobolev in :gh:`133601`.) - wave ---- @@ -1573,6 +1607,21 @@ New deprecations (Contributed by Nikita Sobolev in :gh:`136355`.) +* :mod:`collections.abc` + + * The following statements now cause ``DeprecationWarning``\ s to be emitted + at runtime: + + * ``from collections.abc import ByteString`` + * ``import collections.abc; collections.abc.ByteString``. + + ``DeprecationWarning``\ s were already emitted if + :class:`collections.abc.ByteString` was subclassed or used as the second + argument to :func:`isinstance` or :func:`issubclass`, but warnings were not + previously emitted if it was merely imported or accessed from the + :mod:`!collections.abc` module. + + * :mod:`hashlib`: * In hash function constructors such as :func:`~hashlib.new` or the @@ -1596,6 +1645,22 @@ New deprecations (Contributed by Sergey B Kirpichev and Serhiy Storchaka in :gh:`143715`.) +* :mod:`typing`: + + * The following statements now cause ``DeprecationWarning``\ s to be emitted + at runtime: + + * ``from typing import ByteString`` + * ``import typing; typing.ByteString``. + + ``DeprecationWarning``\ s were already emitted if :class:`typing.ByteString` + was subclassed or used as the second argument to :func:`isinstance` or + :func:`issubclass`, but warnings were not previously emitted if it was + merely imported or accessed from the :mod:`!typing` module. + + * Deprecated :func:`!typing.no_type_check_decorator` has been removed. + (Contributed by Nikita Sobolev in :gh:`133601`.) + * ``__version__`` * The ``__version__``, ``version`` and ``VERSION`` attributes have been @@ -1723,6 +1788,11 @@ New features Python 3.14. (Contributed by Victor Stinner in :gh:`142417`.) +* Add :c:func:`PyUnstable_DumpTraceback` and + :c:func:`PyUnstable_DumpTracebackThreads` functions to safely output Python + stacktraces. + (Contributed by Alex Malyshev in :gh:`145559`.) + Changed C APIs -------------- @@ -1864,6 +1934,24 @@ Deprecated C APIs use the C11 standard ```` :c:macro:`!INFINITY` instead. (Contributed by Sergey B Kirpichev in :gh:`141004`.) +* The following macros are :term:`soft deprecated`: + + - :c:macro:`Py_ALIGNED`: Prefer ``alignas`` instead. + - :c:macro:`PY_FORMAT_SIZE_T`: Use ``"z"`` directly. + - :c:macro:`Py_LL` & :c:macro:`Py_ULL`: + Use standard suffixes, ``LL`` & ``ULL``. + - :c:macro:`PY_LONG_LONG`, :c:macro:`PY_LLONG_MIN`, :c:macro:`PY_LLONG_MAX`, + :c:macro:`PY_ULLONG_MAX`, :c:macro:`PY_INT32_T`, :c:macro:`PY_UINT32_T`, + :c:macro:`PY_INT64_T`, :c:macro:`PY_UINT64_T`, :c:macro:`PY_SIZE_MAX`: + Use C99 types/limits. + - :c:macro:`Py_UNICODE_SIZE`: Use ``sizeof(wchar_t)`` directly. + - :c:macro:`Py_VA_COPY`: Use ``va_copy`` directly. + + The macro :c:macro:`Py_UNICODE_WIDE`, which was scheduled for removal, + is :term:`soft deprecated` instead. + + (Contributed by Petr Viktorin in :gh:`146175`.) + * :c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated since 3.15 and will be removed in 3.20. (Contributed by Sergey B Kirpichev in :gh:`141004`.) @@ -1900,6 +1988,23 @@ Build changes and :option:`-X dev <-X>` is passed to the Python or Python is built in :ref:`debug mode `. (Contributed by Donghee Na in :gh:`141770`.) +.. _whatsnew315-windows-tail-calling-interpreter: + +* 64-bit builds using Visual Studio 2026 (MSVC 18) may now use the new + :ref:`tail-calling interpreter `. + Results on Visual Studio 18.1.1 report between + `15-20% `__ + speedup on the geometric mean of pyperformance on Windows x86-64 over + the switch-case interpreter on an AMD Ryzen 7 5800X. We have + observed speedups ranging from 14% for large pure-Python libraries + to 40% for long-running small pure-Python scripts on Windows. + This was made possible by a new feature introduced in MSVC 18, + which the official Windows 64-bit binaries on python.org__ now use. + (Contributed by Chris Eibl, Ken Jin, and Brandt Bucher in :gh:`143068`. + Special thanks to Steve Dower, and the MSVC team including Hulon Jenkins.) + + __ https://www.python.org/downloads/windows/ + Porting to Python 3.15 ====================== @@ -1939,3 +2044,9 @@ that may require changes to your code. *dest* is now ``'foo'`` instead of ``'f'``. Pass an explicit *dest* argument to preserve the old behavior. (Contributed by Serhiy Storchaka in :gh:`138697`.) + +* Padding of input no longer required in :func:`base64.urlsafe_b64decode`. + Pass a new argument ``padded=True`` or use :func:`base64.b64decode` + with argument ``altchars=b'-_'`` (this works with older Python versions) + to make padding required. + (Contributed by Serhiy Storchaka in :gh:`73613`.) diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 3b13d90f7692cd..48c461d891e861 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -992,12 +992,12 @@ datetime and time offset and timezone name. This makes it easier to create timezone-aware datetime objects:: - >>> from datetime import datetime, timezone + >>> import datetime as dt - >>> datetime.now(timezone.utc) + >>> dt.datetime.now(dt.timezone.utc) datetime.datetime(2010, 12, 8, 21, 4, 2, 923754, tzinfo=datetime.timezone.utc) - >>> datetime.strptime("01/01/2000 12:00 +0000", "%m/%d/%Y %H:%M %z") + >>> dt.datetime.strptime("01/01/2000 12:00 +0000", "%m/%d/%Y %H:%M %z") datetime.datetime(2000, 1, 1, 12, 0, tzinfo=datetime.timezone.utc) * Also, :class:`~datetime.timedelta` objects can now be multiplied by diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 545a17aecab8ee..5078fc30ac111e 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -50,7 +50,6 @@ For full details, see the :ref:`changelog `. .. testsetup:: - from datetime import date from math import cos, radians from unicodedata import normalize import re @@ -208,14 +207,15 @@ subdirectories). Debug build uses the same ABI as release build ----------------------------------------------- -Python now uses the same ABI whether it's built in release or debug mode. On -Unix, when Python is built in debug mode, it is now possible to load C -extensions built in release mode and C extensions built using the stable ABI. +The ABI of Python :ref:`debug builds ` is now compatible with +Python release builds. On Unix, when Python is built in debug mode, it is now +possible to load C extensions built in release mode and C extensions built +using the stable ABI. The inverse is not true, as debug builds expose +additional symbols not available in release builds. -Release builds and :ref:`debug builds ` are now ABI compatible: defining the -``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro, which -introduces the only ABI incompatibility. The ``Py_TRACE_REFS`` macro, which -adds the :func:`sys.getobjects` function and the :envvar:`PYTHONDUMPREFS` +Defining the ``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro, +which introduces the only ABI incompatibility. The ``Py_TRACE_REFS`` macro, +which adds the :func:`sys.getobjects` function and the :envvar:`PYTHONDUMPREFS` environment variable, can be set using the new :option:`./configure --with-trace-refs <--with-trace-refs>` build option. (Contributed by Victor Stinner in :issue:`36465`.) @@ -259,15 +259,16 @@ Added an ``=`` specifier to :term:`f-string`\s. An f-string such as ``f'{expr=}'`` will expand to the text of the expression, an equal sign, then the representation of the evaluated expression. For example: + >>> import datetime as dt >>> user = 'eric_idle' - >>> member_since = date(1975, 7, 31) + >>> member_since = dt.date(1975, 7, 31) >>> f'{user=} {member_since=}' "user='eric_idle' member_since=datetime.date(1975, 7, 31)" The usual :ref:`f-string format specifiers ` allow more control over how the result of the expression is displayed:: - >>> delta = date.today() - member_since + >>> delta = dt.date.today() - member_since >>> f'{user=!s} {delta.days=:,d}' 'user=eric_idle delta.days=16,075' diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 40d4a27bff9fee..49a52b7504bc95 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -282,20 +282,20 @@ the standard library. It adds :class:`zoneinfo.ZoneInfo`, a concrete Example:: >>> from zoneinfo import ZoneInfo - >>> from datetime import datetime, timedelta + >>> import datetime as dt >>> # Daylight saving time - >>> dt = datetime(2020, 10, 31, 12, tzinfo=ZoneInfo("America/Los_Angeles")) - >>> print(dt) + >>> when = dt.datetime(2020, 10, 31, 12, tzinfo=ZoneInfo("America/Los_Angeles")) + >>> print(when) 2020-10-31 12:00:00-07:00 - >>> dt.tzname() + >>> when.tzname() 'PDT' >>> # Standard time - >>> dt += timedelta(days=7) - >>> print(dt) + >>> when += dt.timedelta(days=7) + >>> print(when) 2020-11-07 12:00:00-08:00 - >>> print(dt.tzname()) + >>> print(when.tzname()) PST diff --git a/Include/Python.h b/Include/Python.h index 17cbc083241514..e6e5cab67e2045 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -47,10 +47,6 @@ #endif #if defined(Py_GIL_DISABLED) -# if defined(Py_LIMITED_API) && !defined(_Py_OPAQUE_PYOBJECT) -# error "Py_LIMITED_API is not currently supported in the free-threaded build" -# endif - # if defined(_MSC_VER) # include // __readgsqword() # endif diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index 804c1e9427e063..998ebe6891577e 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -133,6 +133,11 @@ _PyLong_CompactValue(const PyLongObject *op) assert(PyType_HasFeature(op->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); assert(PyUnstable_Long_IsCompact(op)); sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK); + if (sign == 0) { + // gh-147988: Make sure that the digit is zero. + // It helps detecting the usage of uninitialized digits. + assert(op->long_value.ob_digit[0] == 0); + } return sign * (Py_ssize_t)op->long_value.ob_digit[0]; } diff --git a/Include/cpython/modsupport.h b/Include/cpython/modsupport.h index b9f253e06b31c9..cfeee6e8ab3414 100644 --- a/Include/cpython/modsupport.h +++ b/Include/cpython/modsupport.h @@ -39,13 +39,7 @@ PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywordsFast(PyObject *, PyObject *, struct _PyArg_Parser *, ...); #ifdef Py_BUILD_CORE -// Internal; defined here to avoid explicitly including pycore_modsupport.h -#define _Py_INTERNAL_ABI_SLOT \ - {Py_mod_abi, (void*) &(PyABIInfo) { \ - .abiinfo_major_version = 1, \ - .abiinfo_minor_version = 0, \ - .flags = PyABIInfo_INTERNAL, \ - .build_version = PY_VERSION_HEX, \ - .abi_version = PY_VERSION_HEX }} \ - /////////////////////////////////////////////////////// +// For internal use in stdlib. Needs C99 compound literals. +// Defined here to avoid every stdlib module including pycore_modsupport.h +#define _Py_ABI_SLOT {Py_mod_abi, (void*) &(PyABIInfo) _PyABIInfo_DEFAULT} #endif diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index ce907fd6a4c453..e85b360c986668 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -72,8 +72,8 @@ // def _Py_atomic_load_ptr_acquire(obj): // return obj # acquire // -// def _Py_atomic_store_ptr_release(obj): -// return obj # release +// def _Py_atomic_store_ptr_release(obj, value): +// obj = value # release // // def _Py_atomic_fence_seq_cst(): // # sequential consistency @@ -532,6 +532,9 @@ _Py_atomic_store_int_release(int *obj, int value); static inline int _Py_atomic_load_int_acquire(const int *obj); +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value); + static inline void _Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index c045213c898a03..253b35082aafcd 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -580,6 +580,10 @@ static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline int _Py_atomic_load_int_acquire(const int *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 8b9dd3eb0f8e16..3b3c5f7017e957 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -971,12 +971,6 @@ _Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) *(volatile unsigned short *)obj = value; } -static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) -{ - *(volatile unsigned int *)obj = value; -} - static inline void _Py_atomic_store_long_relaxed(long *obj, long value) { @@ -1079,6 +1073,19 @@ _Py_atomic_store_int8_release(int8_t *obj, int8_t value) #endif } +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(volatile unsigned int *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_uint_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index cfc8dbefc63d09..faef303da70314 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -459,7 +459,7 @@ static inline uint16_t _Py_atomic_load_uint16(const uint16_t *obj) { _Py_USING_STD; - return atomic_load((const _Atomic(uint32_t)*)obj); + return atomic_load((const _Atomic(uint16_t)*)obj); } static inline uint32_t diff --git a/Include/cpython/traceback.h b/Include/cpython/traceback.h index 81c51944f136f2..c6a6f3814a2417 100644 --- a/Include/cpython/traceback.h +++ b/Include/cpython/traceback.h @@ -11,3 +11,10 @@ struct _traceback { int tb_lasti; int tb_lineno; }; + +PyAPI_FUNC(const char*) PyUnstable_DumpTraceback(int fd, PyThreadState *tstate); + +PyAPI_FUNC(const char*) PyUnstable_DumpTracebackThreads( + int fd, + PyInterpreterState *interp, + PyThreadState *current_tstate); diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index f27ec4350bb2c8..2479630021bfe3 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -249,16 +249,7 @@ static inline void _Py_LeaveRecursiveCallTstate(PyThreadState *tstate) { PyAPI_FUNC(void) _Py_InitializeRecursionLimits(PyThreadState *tstate); -static inline int _Py_ReachedRecursionLimit(PyThreadState *tstate) { - uintptr_t here_addr = _Py_get_machine_stack_pointer(); - _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - assert(_tstate->c_stack_hard_limit != 0); -#if _Py_STACK_GROWS_DOWN - return here_addr <= _tstate->c_stack_soft_limit; -#else - return here_addr >= _tstate->c_stack_soft_limit; -#endif -} +PyAPI_FUNC(int) _Py_ReachedRecursionLimit(PyThreadState *tstate); // Export for test_peg_generator PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin( @@ -311,7 +302,7 @@ PyAPI_FUNC(int) _PyEval_ExceptionGroupMatch(_PyInterpreterFrame *, PyObject* exc PyAPI_FUNC(void) _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg); PyAPI_FUNC(void) _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj); PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg); -PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs); +PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs, PyObject *dupkey); PyAPI_FUNC(PyObject *) _PyEval_ImportFrom(PyThreadState *, PyObject *, PyObject *); PyAPI_FUNC(PyObject *) _PyEval_LazyImportName( @@ -342,6 +333,7 @@ PyAPI_FUNC(PyObject *) _PyEval_GetAwaitable(PyObject *iterable, int oparg); PyAPI_FUNC(PyObject *) _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *name); PyAPI_FUNC(int) _Py_Check_ArgsIterable(PyThreadState *tstate, PyObject *func, PyObject *args); +PyAPI_FUNC(_PyStackRef) _PyEval_GetIter(_PyStackRef iterable, _PyStackRef *null_or_index, int yield_from); /* * Indicate whether a special method of given 'oparg' can use the (improved) @@ -455,7 +447,7 @@ _Py_BuiltinCallFastWithKeywords_StackRefSteal( PyAPI_FUNC(PyObject *) _PyCallMethodDescriptorFast_StackRefSteal( _PyStackRef callable, - PyMethodDef *meth, + PyCFunctionFast cfunc, PyObject *self, _PyStackRef *arguments, int total_args); @@ -463,7 +455,7 @@ _PyCallMethodDescriptorFast_StackRefSteal( PyAPI_FUNC(PyObject *) _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( _PyStackRef callable, - PyMethodDef *meth, + PyCFunctionFastWithKeywords cfunc, PyObject *self, _PyStackRef *arguments, int total_args); diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 66f14e69f33f44..c166f963da4f66 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -222,6 +222,8 @@ typedef struct _Py_DebugOffsets { uint64_t size; uint64_t collecting; uint64_t frame; + uint64_t generation_stats_size; + uint64_t generation_stats; } gc; // Generator object offset; @@ -373,6 +375,8 @@ typedef struct _Py_DebugOffsets { .size = sizeof(struct _gc_runtime_state), \ .collecting = offsetof(struct _gc_runtime_state, collecting), \ .frame = offsetof(struct _gc_runtime_state, frame), \ + .generation_stats_size = sizeof(struct gc_stats), \ + .generation_stats = offsetof(struct _gc_runtime_state, generation_stats), \ }, \ .gen_object = { \ .size = sizeof(PyGenObject), \ diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 6d7d68eda84c5a..d58539fa846563 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -55,7 +55,7 @@ extern Py_ssize_t _PyDict_SizeOf_LockHeld(PyDictObject *); of a key wins, if override is 2, a KeyError with conflicting key as argument is raised. */ -PyAPI_FUNC(int) _PyDict_MergeEx(PyObject *mp, PyObject *other, int override); +PyAPI_FUNC(int) _PyDict_MergeUniq(PyObject *mp, PyObject *other, PyObject **dupkey); extern void _PyDict_DebugMallocStats(FILE *out); @@ -138,13 +138,14 @@ extern PyObject *_PyDict_LoadBuiltinsFromGlobals(PyObject *globals); /* Consumes references to key and value */ PyAPI_FUNC(int) _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); +PyAPI_FUNC(int) _PyDict_SetItem_Take2_KnownHash(PyDictObject *op, PyObject *key, PyObject *value, Py_hash_t hash); extern int _PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyDict_SetItem_KnownHash_LockHeld(PyDictObject *mp, PyObject *key, PyObject *value, Py_hash_t hash); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyDict_GetItemRef_KnownHash_LockHeld(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); -extern int _PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); +PyAPI_FUNC(int) _PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); extern int _PyDict_GetItemRef_Unicode_LockHeld(PyDictObject *op, PyObject *key, PyObject **result); PyAPI_FUNC(int) _PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, PyObject *name, PyObject *value); diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 4b1e289c6ff468..beae65213a27b6 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1974,6 +1974,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(overlapped)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(owner)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pad)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(padded)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pages)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(parameter)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(parent)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 6ee649b59a5c37..bb1c6dbaf03906 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -697,6 +697,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(overlapped) STRUCT_FOR_ID(owner) STRUCT_FOR_ID(pad) + STRUCT_FOR_ID(padded) STRUCT_FOR_ID(pages) STRUCT_FOR_ID(parameter) STRUCT_FOR_ID(parent) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 4822360a8f08d0..c4b084642668a9 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -177,31 +177,54 @@ struct gc_generation { generations */ }; -struct gc_collection_stats { - /* number of collected objects */ - Py_ssize_t collected; - /* total number of uncollectable objects (put into gc.garbage) */ - Py_ssize_t uncollectable; - // Total number of objects considered for collection and traversed: - Py_ssize_t candidates; - // Duration of the collection in seconds: - double duration; -}; - /* Running stats per generation */ struct gc_generation_stats { + PyTime_t ts_start; + PyTime_t ts_stop; + + /* heap_size on the start of the collection */ + Py_ssize_t heap_size; + + /* work_to_do on the start of the collection */ + Py_ssize_t work_to_do; + /* total number of collections */ Py_ssize_t collections; + + /* total number of visited objects */ + Py_ssize_t object_visits; + /* total number of collected objects */ Py_ssize_t collected; /* total number of uncollectable objects (put into gc.garbage) */ Py_ssize_t uncollectable; // Total number of objects considered for collection and traversed: Py_ssize_t candidates; - // Duration of the collection in seconds: + + Py_ssize_t objects_transitively_reachable; + Py_ssize_t objects_not_transitively_reachable; + + // Total duration of the collection in seconds: double duration; }; +#ifdef Py_GIL_DISABLED +#define GC_YOUNG_STATS_SIZE 1 +#define GC_OLD_STATS_SIZE 1 +#else +#define GC_YOUNG_STATS_SIZE 11 +#define GC_OLD_STATS_SIZE 3 +#endif +struct gc_young_stats_buffer { + struct gc_generation_stats items[GC_YOUNG_STATS_SIZE]; + int8_t index; +}; + +struct gc_old_stats_buffer { + struct gc_generation_stats items[GC_OLD_STATS_SIZE]; + int8_t index; +}; + enum _GCPhase { GC_PHASE_MARK = 0, GC_PHASE_COLLECT = 1 @@ -211,6 +234,11 @@ enum _GCPhase { signature of gc.collect and change the size of PyStats.gc_stats */ #define NUM_GENERATIONS 3 +struct gc_stats { + struct gc_young_stats_buffer young; + struct gc_old_stats_buffer old[2]; +}; + struct _gc_runtime_state { /* Is automatic collection enabled? */ int enabled; @@ -220,7 +248,7 @@ struct _gc_runtime_state { struct gc_generation old[2]; /* a permanent generation which won't be collected */ struct gc_generation permanent_generation; - struct gc_generation_stats generation_stats[NUM_GENERATIONS]; + struct gc_stats *generation_stats; /* true if we are currently running the collector */ int collecting; // The frame that started the current collection. It might be NULL even when diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index 14e2f245834dca..8db1aebdc11401 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -102,10 +102,10 @@ static inline _PyStackRef *_PyFrame_Stackbase(_PyInterpreterFrame *f) { return (f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); } -static inline _PyStackRef _PyFrame_StackPeek(_PyInterpreterFrame *f) { +static inline _PyStackRef _PyFrame_StackPeek(_PyInterpreterFrame *f, int depth) { assert(f->stackpointer > _PyFrame_Stackbase(f)); - assert(!PyStackRef_IsNull(f->stackpointer[-1])); - return f->stackpointer[-1]; + assert(!PyStackRef_IsNull(f->stackpointer[-depth])); + return f->stackpointer[-depth]; } static inline _PyStackRef _PyFrame_StackPop(_PyInterpreterFrame *f) { diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index d545ba0c3abb52..fb5622c99f7a13 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -64,7 +64,8 @@ PyAPI_FUNC(void) _PyLong_ExactDealloc(PyObject *self); # error "_PY_NSMALLPOSINTS must be greater than or equal to 257" #endif -#define _PY_IS_SMALL_INT(val) ((val) >= 0 && (val) < 256 && (val) < _PY_NSMALLPOSINTS) +#define _PY_IS_SMALL_INT(val) \ + (-_PY_NSMALLNEGINTS <= (val) && (val) < _PY_NSMALLPOSINTS) // Return a reference to the immortal zero singleton. // The function cannot return NULL. @@ -231,6 +232,21 @@ _PyLong_IsPositive(const PyLongObject *op) return (op->long_value.lv_tag & SIGN_MASK) == 0; } +/* Return true if the argument is a small int */ +static inline bool +_PyLong_IsSmallInt(const PyLongObject *op) +{ + assert(PyLong_Check(op)); + bool is_small_int = (op->long_value.lv_tag & IMMORTALITY_BIT_MASK) != 0; + if (is_small_int) { + assert(PyLong_CheckExact(op)); + assert(_Py_IsImmortal(op)); + assert((_PyLong_IsCompact(op) + && _PY_IS_SMALL_INT(_PyLong_CompactValue(op)))); + } + return is_small_int; +} + static inline Py_ssize_t _PyLong_DigitCount(const PyLongObject *op) { @@ -270,6 +286,14 @@ _PyLong_SameSign(const PyLongObject *a, const PyLongObject *b) return (a->long_value.lv_tag & SIGN_MASK) == (b->long_value.lv_tag & SIGN_MASK); } +/* Initialize the tag of a freshly-allocated int. */ +static inline void +_PyLong_InitTag(PyLongObject *op) +{ + assert(PyLong_Check(op)); + op->long_value.lv_tag = SIGN_ZERO; /* non-immortal zero */ +} + #define TAG_FROM_SIGN_AND_SIZE(sign, size) \ ((uintptr_t)(1 - (sign)) | ((uintptr_t)(size) << NON_SIZE_BITS)) @@ -279,6 +303,7 @@ _PyLong_SetSignAndDigitCount(PyLongObject *op, int sign, Py_ssize_t size) assert(size >= 0); assert(-1 <= sign && sign <= 1); assert(sign != 0 || size == 0); + assert(!_PyLong_IsSmallInt(op)); op->long_value.lv_tag = TAG_FROM_SIGN_AND_SIZE(sign, size); } @@ -286,13 +311,16 @@ static inline void _PyLong_SetDigitCount(PyLongObject *op, Py_ssize_t size) { assert(size >= 0); + assert(!_PyLong_IsSmallInt(op)); op->long_value.lv_tag = (((size_t)size) << NON_SIZE_BITS) | (op->long_value.lv_tag & SIGN_MASK); } #define NON_SIZE_MASK ~(uintptr_t)((1 << NON_SIZE_BITS) - 1) static inline void -_PyLong_FlipSign(PyLongObject *op) { +_PyLong_FlipSign(PyLongObject *op) +{ + assert(!_PyLong_IsSmallInt(op)); unsigned int flipped_sign = 2 - (op->long_value.lv_tag & SIGN_MASK); op->long_value.lv_tag &= NON_SIZE_MASK; op->long_value.lv_tag |= flipped_sign; diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index ec9cfe432371c7..9d36165c8a8ffb 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -292,7 +292,8 @@ Known values: Python 3.15a4 3659 (Add CALL_FUNCTION_EX specialization) Python 3.15a4 3660 (Change generator preamble code) Python 3.15a4 3661 (Lazy imports IMPORT_NAME opcode changes) - Python 3.15a6 3662 (Add counter to RESUME) + Python 3.15a8 3662 (Add counter to RESUME) + Python 3.15a8 3663 (Merge GET_ITER and GET_YIELD_FROM_ITER. Modify SEND to make it a bit more like FOR_ITER) Python 3.16 will start with 3700 @@ -306,7 +307,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3662 +#define PYC_MAGIC_NUMBER 3663 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 96685af02cd0e3..de701ced675cd4 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -144,7 +144,7 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) new_refcnt = _Py_IMMORTAL_INITIAL_REFCNT; } # if SIZEOF_VOID_P > 4 - op->ob_refcnt = (PY_UINT32_T)new_refcnt; + op->ob_refcnt = (uint32_t)new_refcnt; # else op->ob_refcnt = new_refcnt; # endif diff --git a/Include/internal/pycore_obmalloc.h b/Include/internal/pycore_obmalloc.h index 0b23bb48dd5c1b..d4dbe541e6da51 100644 --- a/Include/internal/pycore_obmalloc.h +++ b/Include/internal/pycore_obmalloc.h @@ -691,7 +691,11 @@ struct _obmalloc_state { /* Allocate memory directly from the O/S virtual memory system, - * where supported. Otherwise fallback on malloc */ + * where supported. Otherwise fallback on malloc. + * + * Large-page and huge-page backends may round the mapped size up + * internally, so pass the original requested size back to + * _PyObject_VirtualFree(). */ void *_PyObject_VirtualAlloc(size_t size); void _PyObject_VirtualFree(void *, size_t size); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 2ff6e75e28a74c..916f3b8ee863c4 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -157,7 +157,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case CHECK_EXC_MATCH: return 2; case CLEANUP_THROW: - return 3; + return 4; case COMPARE_OP: return 2; case COMPARE_OP_FLOAT: @@ -199,7 +199,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case END_FOR: return 1; case END_SEND: - return 2; + return 3; case ENTER_EXECUTOR: return 0; case EXIT_INIT_CHECK: @@ -230,8 +230,6 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case GET_LEN: return 1; - case GET_YIELD_FROM_ITER: - return 1; case IMPORT_FROM: return 1; case IMPORT_NAME: @@ -247,7 +245,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case INSTRUMENTED_END_FOR: return 3; case INSTRUMENTED_END_SEND: - return 2; + return 3; case INSTRUMENTED_FOR_ITER: return 2; case INSTRUMENTED_INSTRUCTION: @@ -433,9 +431,9 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case RETURN_VALUE: return 1; case SEND: - return 2; + return 3; case SEND_GEN: - return 2; + return 3; case SETUP_ANNOTATIONS: return 0; case SETUP_CLEANUP: @@ -650,7 +648,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case CHECK_EXC_MATCH: return 2; case CLEANUP_THROW: - return 2; + return 3; case COMPARE_OP: return 1; case COMPARE_OP_FLOAT: @@ -723,8 +721,6 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 2; case GET_LEN: return 2; - case GET_YIELD_FROM_ITER: - return 1; case IMPORT_FROM: return 2; case IMPORT_NAME: @@ -926,9 +922,9 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case RETURN_VALUE: return 1; case SEND: - return 2; + return 3; case SEND_GEN: - return 1; + return 2; case SETUP_ANNOTATIONS: return 0; case SETUP_CLEANUP: @@ -1135,7 +1131,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [CALL_EX_PY] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_FUNCTION_EX] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_ISINSTANCE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, @@ -1172,8 +1168,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, @@ -1190,9 +1186,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_ITER] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, @@ -1224,7 +1219,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [JUMP_BACKWARD_NO_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, @@ -1287,7 +1282,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [RESUME_CHECK_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1380,23 +1375,23 @@ _PyOpcode_macro_expansion[256] = { [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 11, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_BOUND_METHOD_GENERAL] = { .nuops = 8, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_BUILTIN_CLASS] = { .nuops = 3, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CALL_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_BUILTIN_FAST] = { .nuops = 3, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 3, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_BUILTIN_O] = { .nuops = 5, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_FAST] = { .nuops = 4, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 4, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_O] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_O, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_EX_NON_PY_GENERAL] = { .nuops = 4, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CALL_FUNCTION_EX_NON_PY_GENERAL, OPARG_SIMPLE, 1 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 1 } } }, [CALL_EX_PY] = { .nuops = 7, .uops = { { _RECORD_4OS, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CHECK_IS_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _PY_FRAME_EX, OPARG_SIMPLE, 1 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, [CALL_INTRINSIC_1] = { .nuops = 2, .uops = { { _CALL_INTRINSIC_1, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, - [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, OPARG_SIMPLE, 0 } } }, + [CALL_INTRINSIC_2] = { .nuops = 3, .uops = { { _CALL_INTRINSIC_2, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [CALL_ISINSTANCE] = { .nuops = 3, .uops = { { _GUARD_THIRD_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_ISINSTANCE, OPARG_SIMPLE, 3 }, { _CALL_ISINSTANCE, OPARG_SIMPLE, 3 } } }, [CALL_KW_BOUND_METHOD] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, OPARG_SIMPLE, 3 }, { _CALL_KW_NON_PY, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_KW_PY] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_LEN] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_LEN, OPARG_SIMPLE, 3 }, { _CALL_LEN, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [CALL_LIST_APPEND] = { .nuops = 6, .uops = { { _GUARD_CALLABLE_LIST_APPEND, OPARG_SIMPLE, 3 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 3 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 3 }, { _CALL_LIST_APPEND, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 3, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 3, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CALL_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 3, .uops = { { _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 4, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 5, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 8, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_NON_PY_GENERAL] = { .nuops = 4, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_IS_NOT_PY_CALLABLE, OPARG_SIMPLE, 3 }, { _CALL_NON_PY_GENERAL, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_PY_EXACT_ARGS] = { .nuops = 9, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_PY_GENERAL] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, @@ -1421,8 +1416,8 @@ _PyOpcode_macro_expansion[256] = { [DELETE_GLOBAL] = { .nuops = 1, .uops = { { _DELETE_GLOBAL, OPARG_SIMPLE, 0 } } }, [DELETE_NAME] = { .nuops = 1, .uops = { { _DELETE_NAME, OPARG_SIMPLE, 0 } } }, [DELETE_SUBSCR] = { .nuops = 1, .uops = { { _DELETE_SUBSCR, OPARG_SIMPLE, 0 } } }, - [DICT_MERGE] = { .nuops = 1, .uops = { { _DICT_MERGE, OPARG_SIMPLE, 0 } } }, - [DICT_UPDATE] = { .nuops = 1, .uops = { { _DICT_UPDATE, OPARG_SIMPLE, 0 } } }, + [DICT_MERGE] = { .nuops = 2, .uops = { { _DICT_MERGE, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [DICT_UPDATE] = { .nuops = 2, .uops = { { _DICT_UPDATE, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [END_FOR] = { .nuops = 1, .uops = { { _END_FOR, OPARG_SIMPLE, 0 } } }, [END_SEND] = { .nuops = 1, .uops = { { _END_SEND, OPARG_SIMPLE, 0 } } }, [EXIT_INIT_CHECK] = { .nuops = 1, .uops = { { _EXIT_INIT_CHECK, OPARG_SIMPLE, 0 } } }, @@ -1438,7 +1433,6 @@ _PyOpcode_macro_expansion[256] = { [GET_AWAITABLE] = { .nuops = 1, .uops = { { _GET_AWAITABLE, OPARG_SIMPLE, 0 } } }, [GET_ITER] = { .nuops = 1, .uops = { { _GET_ITER, OPARG_SIMPLE, 0 } } }, [GET_LEN] = { .nuops = 1, .uops = { { _GET_LEN, OPARG_SIMPLE, 0 } } }, - [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { _GET_YIELD_FROM_ITER, OPARG_SIMPLE, 0 } } }, [IMPORT_FROM] = { .nuops = 1, .uops = { { _IMPORT_FROM, OPARG_SIMPLE, 0 } } }, [IMPORT_NAME] = { .nuops = 1, .uops = { { _IMPORT_NAME, OPARG_SIMPLE, 0 } } }, [IS_OP] = { .nuops = 3, .uops = { { _IS_OP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, @@ -1446,7 +1440,7 @@ _PyOpcode_macro_expansion[256] = { [JUMP_BACKWARD_NO_INTERRUPT] = { .nuops = 1, .uops = { { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 0 } } }, [JUMP_BACKWARD_NO_JIT] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, OPARG_SIMPLE, 0 } } }, - [LIST_EXTEND] = { .nuops = 1, .uops = { { _LIST_EXTEND, OPARG_SIMPLE, 0 } } }, + [LIST_EXTEND] = { .nuops = 2, .uops = { { _LIST_EXTEND, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, OPARG_SIMPLE, 8 } } }, [LOAD_ATTR_CLASS] = { .nuops = 3, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_CLASS, 2, 3 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, @@ -1501,7 +1495,7 @@ _PyOpcode_macro_expansion[256] = { [RESUME_CHECK] = { .nuops = 1, .uops = { { _RESUME_CHECK, OPARG_SIMPLE, 1 } } }, [RETURN_GENERATOR] = { .nuops = 1, .uops = { { _RETURN_GENERATOR, OPARG_SIMPLE, 0 } } }, [RETURN_VALUE] = { .nuops = 2, .uops = { { _MAKE_HEAP_SAFE, OPARG_SIMPLE, 0 }, { _RETURN_VALUE, OPARG_SIMPLE, 0 } } }, - [SEND_GEN] = { .nuops = 4, .uops = { { _RECORD_NOS_GEN_FUNC, OPARG_SIMPLE, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _SEND_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, + [SEND_GEN] = { .nuops = 4, .uops = { { _RECORD_3OS_GEN_FUNC, OPARG_SIMPLE, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _SEND_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, [SETUP_ANNOTATIONS] = { .nuops = 1, .uops = { { _SETUP_ANNOTATIONS, OPARG_SIMPLE, 0 } } }, [SET_ADD] = { .nuops = 1, .uops = { { _SET_ADD, OPARG_SIMPLE, 0 } } }, [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, OPARG_SIMPLE, 0 } } }, @@ -1641,7 +1635,6 @@ const char *_PyOpcode_OpName[267] = { [GET_AWAITABLE] = "GET_AWAITABLE", [GET_ITER] = "GET_ITER", [GET_LEN] = "GET_LEN", - [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [IMPORT_FROM] = "IMPORT_FROM", [IMPORT_NAME] = "IMPORT_NAME", [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", @@ -1818,6 +1811,7 @@ const uint8_t _PyOpcode_Caches[256] = { PyAPI_DATA(const uint8_t) _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { + [120] = 120, [121] = 121, [122] = 122, [123] = 123, @@ -1940,7 +1934,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [GET_AWAITABLE] = GET_AWAITABLE, [GET_ITER] = GET_ITER, [GET_LEN] = GET_LEN, - [GET_YIELD_FROM_ITER] = GET_YIELD_FROM_ITER, [IMPORT_FROM] = IMPORT_FROM, [IMPORT_NAME] = IMPORT_NAME, [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, @@ -2079,6 +2072,7 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ + case 120: \ case 121: \ case 122: \ case 123: \ diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index e4d859fcc47d02..69178381993fba 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -86,6 +86,10 @@ extern "C" { #define RESUME_OPARG_LOCATION_MASK 0x3 #define RESUME_OPARG_DEPTH1_MASK 0x4 +#define GET_ITER_YIELD_FROM 1 +#define GET_ITER_YIELD_FROM_NO_CHECK 2 +#define GET_ITER_YIELD_FROM_CORO_CHECK 3 + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 94c79edc855bbf..cf01c620476ff7 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -290,9 +290,13 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) #define REF_IS_BORROWED 1 -#define REF_IS_INVALID 2 +#define REF_IS_UNIQUE 2 +#define REF_IS_INVALID 3 #define REF_TAG_BITS 3 +#define REF_GET_TAG(x) ((uintptr_t)(x) & (REF_TAG_BITS)) +#define REF_CLEAR_TAG(x) ((uintptr_t)(x) & (~REF_TAG_BITS)) + #define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~REF_TAG_BITS))) static inline JitOptSymbol * @@ -313,13 +317,34 @@ PyJitRef_Wrap(JitOptSymbol *sym) static inline JitOptRef PyJitRef_WrapInvalid(void *ptr) { - return (JitOptRef){.bits=(uintptr_t)ptr | REF_IS_INVALID}; + return (JitOptRef){.bits = REF_CLEAR_TAG((uintptr_t)ptr) | REF_IS_INVALID}; } static inline bool PyJitRef_IsInvalid(JitOptRef ref) { - return (ref.bits & REF_IS_INVALID) == REF_IS_INVALID; + return REF_GET_TAG(ref.bits) == REF_IS_INVALID; +} + +static inline JitOptRef +PyJitRef_MakeUnique(JitOptRef ref) +{ + return (JitOptRef){ REF_CLEAR_TAG(ref.bits) | REF_IS_UNIQUE }; +} + +static inline bool +PyJitRef_IsUnique(JitOptRef ref) +{ + return REF_GET_TAG(ref.bits) == REF_IS_UNIQUE; +} + +static inline JitOptRef +PyJitRef_StripBorrowInfo(JitOptRef ref) +{ + if (PyJitRef_IsUnique(ref)) { + return ref; + } + return (JitOptRef){ .bits = REF_CLEAR_TAG(ref.bits) }; } static inline JitOptRef @@ -328,10 +353,19 @@ PyJitRef_StripReferenceInfo(JitOptRef ref) return PyJitRef_Wrap(PyJitRef_Unwrap(ref)); } +static inline JitOptRef +PyJitRef_RemoveUnique(JitOptRef ref) +{ + if (PyJitRef_IsUnique(ref)) { + ref = PyJitRef_StripReferenceInfo(ref); + } + return ref; +} + static inline JitOptRef PyJitRef_Borrow(JitOptRef ref) { - return (JitOptRef){ .bits = ref.bits | REF_IS_BORROWED }; + return (JitOptRef){ .bits = REF_CLEAR_TAG(ref.bits) | REF_IS_BORROWED }; } static const JitOptRef PyJitRef_NULL = {.bits = REF_IS_BORROWED}; @@ -345,7 +379,7 @@ PyJitRef_IsNull(JitOptRef ref) static inline int PyJitRef_IsBorrowed(JitOptRef ref) { - return (ref.bits & REF_IS_BORROWED) == REF_IS_BORROWED; + return REF_GET_TAG(ref.bits) == REF_IS_BORROWED; } extern bool _Py_uop_sym_is_null(JitOptRef sym); @@ -360,6 +394,7 @@ extern JitOptRef _Py_uop_sym_new_type( extern JitOptRef _Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val); extern JitOptRef _Py_uop_sym_new_const_steal(JitOptContext *ctx, PyObject *const_val); bool _Py_uop_sym_is_safe_const(JitOptContext *ctx, JitOptRef sym); +bool _Py_uop_sym_is_not_container(JitOptRef sym); _PyStackRef _Py_uop_sym_get_const_as_stackref(JitOptContext *ctx, JitOptRef sym); extern JitOptRef _Py_uop_sym_new_null(JitOptContext *ctx); extern bool _Py_uop_sym_has_type(JitOptRef sym); diff --git a/Include/internal/pycore_optimizer_types.h b/Include/internal/pycore_optimizer_types.h index fe45775201b098..8ecfbea387460b 100644 --- a/Include/internal/pycore_optimizer_types.h +++ b/Include/internal/pycore_optimizer_types.h @@ -147,6 +147,7 @@ typedef struct _Py_UOpsAbstractFrame { int stack_len; int locals_len; bool caller; // We have made a call from this frame during the trace + bool is_c_recursion_checked; JitOptRef callable; PyFunctionObject *func; PyCodeObject *code; diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 778db946c2a3aa..64b029797ab9b3 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1972,6 +1972,7 @@ extern "C" { INIT_ID(overlapped), \ INIT_ID(owner), \ INIT_ID(pad), \ + INIT_ID(padded), \ INIT_ID(pages), \ INIT_ID(parameter), \ INIT_ID(parent), \ diff --git a/Include/internal/pycore_strhex.h b/Include/internal/pycore_strhex.h index 225f423912f2c2..656acae960ac0d 100644 --- a/Include/internal/pycore_strhex.h +++ b/Include/internal/pycore_strhex.h @@ -10,28 +10,24 @@ extern "C" { // Returns a str() containing the hex representation of argbuf. // Export for '_hashlib' shared extension. -PyAPI_FUNC(PyObject*) _Py_strhex(const - char* argbuf, - const Py_ssize_t arglen); +PyAPI_FUNC(PyObject *) _Py_strhex(const char *argbuf, Py_ssize_t arglen); // Returns a bytes() containing the ASCII hex representation of argbuf. -extern PyObject* _Py_strhex_bytes( - const char* argbuf, - const Py_ssize_t arglen); +extern PyObject *_Py_strhex_bytes(const char *argbuf, Py_ssize_t arglen); // These variants include support for a separator between every N bytes: -extern PyObject* _Py_strhex_with_sep( - const char* argbuf, - const Py_ssize_t arglen, - PyObject* sep, - const int bytes_per_group); +extern PyObject *_Py_strhex_with_sep( + const char *argbuf, + Py_ssize_t arglen, + PyObject *sep, + Py_ssize_t bytes_per_group); // Export for 'binascii' shared extension -PyAPI_FUNC(PyObject*) _Py_strhex_bytes_with_sep( - const char* argbuf, - const Py_ssize_t arglen, - PyObject* sep, - const int bytes_per_group); +PyAPI_FUNC(PyObject *) _Py_strhex_bytes_with_sep( + const char *argbuf, + Py_ssize_t arglen, + PyObject *sep, + Py_ssize_t bytes_per_group); #ifdef __cplusplus } diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index 8357cce9d899fb..e7729965b71769 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -14,55 +14,6 @@ PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int, int *, P // Export for 'pyexact' shared extension PyAPI_FUNC(void) _PyTraceback_Add(const char *, const char *, int); -/* Write the Python traceback into the file 'fd'. For example: - - Traceback (most recent call first): - File "xxx", line xxx in - File "xxx", line xxx in - ... - File "xxx", line xxx in - - This function is written for debug purpose only, to dump the traceback in - the worst case: after a segmentation fault, at fatal error, etc. That's why, - it is very limited. Strings are truncated to 100 characters and encoded to - ASCII with backslashreplace. It doesn't write the source code, only the - function name, filename and line number of each frame. Write only the first - 100 frames: if the traceback is truncated, write the line " ...". - - This function is signal safe. */ - -extern void _Py_DumpTraceback( - int fd, - PyThreadState *tstate); - -/* Write the traceback of all threads into the file 'fd'. current_thread can be - NULL. - - Return NULL on success, or an error message on error. - - This function is written for debug purpose only. It calls - _Py_DumpTraceback() for each thread, and so has the same limitations. It - only write the traceback of the first 100 threads: write "..." if there are - more threads. - - If current_tstate is NULL, the function tries to get the Python thread state - of the current thread. It is not an error if the function is unable to get - the current Python thread state. - - If interp is NULL, the function tries to get the interpreter state from - the current Python thread state, or from - _PyGILState_GetInterpreterStateUnsafe() in last resort. - - It is better to pass NULL to interp and current_tstate, the function tries - different options to retrieve this information. - - This function is signal safe. */ - -extern const char* _Py_DumpTracebackThreads( - int fd, - PyInterpreterState *interp, - PyThreadState *current_tstate); - /* Write a Unicode object into the file descriptor fd. Encode the string to ASCII using the backslashreplace error handler. diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index b3fa28f5bf21d5..9409ec94976d3a 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -21,6 +21,8 @@ extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *); /* other API */ +PyAPI_FUNC(void) _PyStolenTuple_Free(PyObject *self); + #define _PyTuple_ITEMS(op) _Py_RVALUE(_PyTuple_CAST(op)->ob_item) PyAPI_FUNC(PyObject *)_PyTuple_FromStackRefStealOnSuccess(const union _PyStackRef *, Py_ssize_t); diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index bd8f50ff0ee732..461ee36dcebb6d 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -2568,6 +2568,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(padded); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pages); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 50f137fd49fe7e..dd319778b1f2e8 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -13,23 +13,36 @@ extern "C" { #define _SET_IP 301 #define _BINARY_OP 302 #define _BINARY_OP_ADD_FLOAT 303 -#define _BINARY_OP_ADD_INT 304 -#define _BINARY_OP_ADD_UNICODE 305 -#define _BINARY_OP_EXTEND 306 -#define _BINARY_OP_INPLACE_ADD_UNICODE 307 -#define _BINARY_OP_MULTIPLY_FLOAT 308 -#define _BINARY_OP_MULTIPLY_INT 309 -#define _BINARY_OP_SUBSCR_CHECK_FUNC 310 -#define _BINARY_OP_SUBSCR_DICT 311 -#define _BINARY_OP_SUBSCR_INIT_CALL 312 -#define _BINARY_OP_SUBSCR_LIST_INT 313 -#define _BINARY_OP_SUBSCR_LIST_SLICE 314 -#define _BINARY_OP_SUBSCR_STR_INT 315 -#define _BINARY_OP_SUBSCR_TUPLE_INT 316 -#define _BINARY_OP_SUBSCR_USTR_INT 317 -#define _BINARY_OP_SUBTRACT_FLOAT 318 -#define _BINARY_OP_SUBTRACT_INT 319 -#define _BINARY_SLICE 320 +#define _BINARY_OP_ADD_FLOAT_INPLACE 304 +#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT 305 +#define _BINARY_OP_ADD_INT 306 +#define _BINARY_OP_ADD_INT_INPLACE 307 +#define _BINARY_OP_ADD_INT_INPLACE_RIGHT 308 +#define _BINARY_OP_ADD_UNICODE 309 +#define _BINARY_OP_EXTEND 310 +#define _BINARY_OP_INPLACE_ADD_UNICODE 311 +#define _BINARY_OP_MULTIPLY_FLOAT 312 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE 313 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT 314 +#define _BINARY_OP_MULTIPLY_INT 315 +#define _BINARY_OP_MULTIPLY_INT_INPLACE 316 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT 317 +#define _BINARY_OP_SUBSCR_CHECK_FUNC 318 +#define _BINARY_OP_SUBSCR_DICT 319 +#define _BINARY_OP_SUBSCR_DICT_KNOWN_HASH 320 +#define _BINARY_OP_SUBSCR_INIT_CALL 321 +#define _BINARY_OP_SUBSCR_LIST_INT 322 +#define _BINARY_OP_SUBSCR_LIST_SLICE 323 +#define _BINARY_OP_SUBSCR_STR_INT 324 +#define _BINARY_OP_SUBSCR_TUPLE_INT 325 +#define _BINARY_OP_SUBSCR_USTR_INT 326 +#define _BINARY_OP_SUBTRACT_FLOAT 327 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE 328 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT 329 +#define _BINARY_OP_SUBTRACT_INT 330 +#define _BINARY_OP_SUBTRACT_INT_INPLACE 331 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT 332 +#define _BINARY_SLICE 333 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST #define _BUILD_MAP BUILD_MAP @@ -38,175 +51,186 @@ extern "C" { #define _BUILD_STRING BUILD_STRING #define _BUILD_TEMPLATE BUILD_TEMPLATE #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL_BUILTIN_CLASS 321 -#define _CALL_BUILTIN_FAST 322 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 323 -#define _CALL_BUILTIN_O 324 -#define _CALL_FUNCTION_EX_NON_PY_GENERAL 325 -#define _CALL_INTRINSIC_1 326 -#define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 -#define _CALL_ISINSTANCE 327 -#define _CALL_KW_NON_PY 328 -#define _CALL_LEN 329 -#define _CALL_LIST_APPEND 330 -#define _CALL_METHOD_DESCRIPTOR_FAST 331 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 332 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 333 -#define _CALL_METHOD_DESCRIPTOR_O 334 -#define _CALL_NON_PY_GENERAL 335 -#define _CALL_STR_1 336 -#define _CALL_TUPLE_1 337 -#define _CALL_TYPE_1 338 -#define _CHECK_AND_ALLOCATE_OBJECT 339 -#define _CHECK_ATTR_CLASS 340 -#define _CHECK_ATTR_METHOD_LAZY_DICT 341 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 342 +#define _CALL_BUILTIN_CLASS 334 +#define _CALL_BUILTIN_FAST 335 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 336 +#define _CALL_BUILTIN_O 337 +#define _CALL_FUNCTION_EX_NON_PY_GENERAL 338 +#define _CALL_INTRINSIC_1 339 +#define _CALL_INTRINSIC_2 340 +#define _CALL_ISINSTANCE 341 +#define _CALL_KW_NON_PY 342 +#define _CALL_LEN 343 +#define _CALL_LIST_APPEND 344 +#define _CALL_METHOD_DESCRIPTOR_FAST 345 +#define _CALL_METHOD_DESCRIPTOR_FAST_INLINE 346 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 347 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE 348 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 349 +#define _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE 350 +#define _CALL_METHOD_DESCRIPTOR_O 351 +#define _CALL_METHOD_DESCRIPTOR_O_INLINE 352 +#define _CALL_NON_PY_GENERAL 353 +#define _CALL_STR_1 354 +#define _CALL_TUPLE_1 355 +#define _CALL_TYPE_1 356 +#define _CHECK_AND_ALLOCATE_OBJECT 357 +#define _CHECK_ATTR_CLASS 358 +#define _CHECK_ATTR_METHOD_LAZY_DICT 359 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 360 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION_EXACT_ARGS 343 -#define _CHECK_FUNCTION_VERSION 344 -#define _CHECK_FUNCTION_VERSION_INLINE 345 -#define _CHECK_FUNCTION_VERSION_KW 346 -#define _CHECK_IS_NOT_PY_CALLABLE 347 -#define _CHECK_IS_NOT_PY_CALLABLE_EX 348 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 349 -#define _CHECK_IS_PY_CALLABLE_EX 350 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 351 -#define _CHECK_METHOD_VERSION 352 -#define _CHECK_METHOD_VERSION_KW 353 -#define _CHECK_PEP_523 354 -#define _CHECK_PERIODIC 355 -#define _CHECK_PERIODIC_AT_END 356 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 357 -#define _CHECK_RECURSION_REMAINING 358 -#define _CHECK_STACK_SPACE 359 -#define _CHECK_STACK_SPACE_OPERAND 360 -#define _CHECK_VALIDITY 361 -#define _COLD_DYNAMIC_EXIT 362 -#define _COLD_EXIT 363 -#define _COMPARE_OP 364 -#define _COMPARE_OP_FLOAT 365 -#define _COMPARE_OP_INT 366 -#define _COMPARE_OP_STR 367 -#define _CONTAINS_OP 368 -#define _CONTAINS_OP_DICT 369 -#define _CONTAINS_OP_SET 370 +#define _CHECK_FUNCTION_EXACT_ARGS 361 +#define _CHECK_FUNCTION_VERSION 362 +#define _CHECK_FUNCTION_VERSION_INLINE 363 +#define _CHECK_FUNCTION_VERSION_KW 364 +#define _CHECK_IS_NOT_PY_CALLABLE 365 +#define _CHECK_IS_NOT_PY_CALLABLE_EX 366 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 367 +#define _CHECK_IS_PY_CALLABLE_EX 368 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 369 +#define _CHECK_METHOD_VERSION 370 +#define _CHECK_METHOD_VERSION_KW 371 +#define _CHECK_PEP_523 372 +#define _CHECK_PERIODIC 373 +#define _CHECK_PERIODIC_AT_END 374 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 375 +#define _CHECK_RECURSION_LIMIT 376 +#define _CHECK_RECURSION_REMAINING 377 +#define _CHECK_STACK_SPACE 378 +#define _CHECK_STACK_SPACE_OPERAND 379 +#define _CHECK_VALIDITY 380 +#define _COLD_DYNAMIC_EXIT 381 +#define _COLD_EXIT 382 +#define _COMPARE_OP 383 +#define _COMPARE_OP_FLOAT 384 +#define _COMPARE_OP_INT 385 +#define _COMPARE_OP_STR 386 +#define _CONTAINS_OP 387 +#define _CONTAINS_OP_DICT 388 +#define _CONTAINS_OP_SET 389 #define _CONVERT_VALUE CONVERT_VALUE -#define _COPY 371 -#define _COPY_1 372 -#define _COPY_2 373 -#define _COPY_3 374 +#define _COPY 390 +#define _COPY_1 391 +#define _COPY_2 392 +#define _COPY_3 393 #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 375 +#define _CREATE_INIT_FRAME 394 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 376 -#define _DICT_MERGE DICT_MERGE -#define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 377 -#define _DO_CALL_FUNCTION_EX 378 -#define _DO_CALL_KW 379 -#define _DYNAMIC_EXIT 380 +#define _DEOPT 395 +#define _DICT_MERGE 396 +#define _DICT_UPDATE 397 +#define _DO_CALL 398 +#define _DO_CALL_FUNCTION_EX 399 +#define _DO_CALL_KW 400 +#define _DYNAMIC_EXIT 401 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 381 +#define _ERROR_POP_N 402 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 382 -#define _EXPAND_METHOD_KW 383 -#define _FATAL_ERROR 384 +#define _EXPAND_METHOD 403 +#define _EXPAND_METHOD_KW 404 +#define _FATAL_ERROR 405 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 385 -#define _FOR_ITER_GEN_FRAME 386 -#define _FOR_ITER_TIER_TWO 387 +#define _FOR_ITER 406 +#define _FOR_ITER_GEN_FRAME 407 +#define _FOR_ITER_TIER_TWO 408 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN -#define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 388 -#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS 389 -#define _GUARD_BIT_IS_SET_POP 390 -#define _GUARD_BIT_IS_SET_POP_4 391 -#define _GUARD_BIT_IS_SET_POP_5 392 -#define _GUARD_BIT_IS_SET_POP_6 393 -#define _GUARD_BIT_IS_SET_POP_7 394 -#define _GUARD_BIT_IS_UNSET_POP 395 -#define _GUARD_BIT_IS_UNSET_POP_4 396 -#define _GUARD_BIT_IS_UNSET_POP_5 397 -#define _GUARD_BIT_IS_UNSET_POP_6 398 -#define _GUARD_BIT_IS_UNSET_POP_7 399 -#define _GUARD_CALLABLE_ISINSTANCE 400 -#define _GUARD_CALLABLE_LEN 401 -#define _GUARD_CALLABLE_LIST_APPEND 402 -#define _GUARD_CALLABLE_STR_1 403 -#define _GUARD_CALLABLE_TUPLE_1 404 -#define _GUARD_CALLABLE_TYPE_1 405 -#define _GUARD_CODE_VERSION_RETURN_GENERATOR 406 -#define _GUARD_CODE_VERSION_RETURN_VALUE 407 -#define _GUARD_CODE_VERSION_YIELD_VALUE 408 -#define _GUARD_CODE_VERSION__PUSH_FRAME 409 -#define _GUARD_DORV_NO_DICT 410 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 411 -#define _GUARD_GLOBALS_VERSION 412 -#define _GUARD_IP_RETURN_GENERATOR 413 -#define _GUARD_IP_RETURN_VALUE 414 -#define _GUARD_IP_YIELD_VALUE 415 -#define _GUARD_IP__PUSH_FRAME 416 -#define _GUARD_IS_FALSE_POP 417 -#define _GUARD_IS_NONE_POP 418 -#define _GUARD_IS_NOT_NONE_POP 419 -#define _GUARD_IS_TRUE_POP 420 -#define _GUARD_KEYS_VERSION 421 -#define _GUARD_NOS_ANY_DICT 422 -#define _GUARD_NOS_COMPACT_ASCII 423 -#define _GUARD_NOS_DICT 424 -#define _GUARD_NOS_FLOAT 425 -#define _GUARD_NOS_INT 426 -#define _GUARD_NOS_LIST 427 -#define _GUARD_NOS_NOT_NULL 428 -#define _GUARD_NOS_NULL 429 -#define _GUARD_NOS_OVERFLOWED 430 -#define _GUARD_NOS_TUPLE 431 -#define _GUARD_NOS_UNICODE 432 -#define _GUARD_NOT_EXHAUSTED_LIST 433 -#define _GUARD_NOT_EXHAUSTED_RANGE 434 -#define _GUARD_NOT_EXHAUSTED_TUPLE 435 -#define _GUARD_THIRD_NULL 436 -#define _GUARD_TOS_ANY_DICT 437 -#define _GUARD_TOS_ANY_SET 438 -#define _GUARD_TOS_DICT 439 -#define _GUARD_TOS_FLOAT 440 -#define _GUARD_TOS_FROZENDICT 441 -#define _GUARD_TOS_FROZENSET 442 -#define _GUARD_TOS_INT 443 -#define _GUARD_TOS_LIST 444 -#define _GUARD_TOS_OVERFLOWED 445 -#define _GUARD_TOS_SET 446 -#define _GUARD_TOS_SLICE 447 -#define _GUARD_TOS_TUPLE 448 -#define _GUARD_TOS_UNICODE 449 -#define _GUARD_TYPE_VERSION 450 -#define _GUARD_TYPE_VERSION_LOCKED 451 -#define _HANDLE_PENDING_AND_DEOPT 452 +#define _GUARD_BINARY_OP_EXTEND 409 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS 410 +#define _GUARD_BIT_IS_SET_POP 411 +#define _GUARD_BIT_IS_SET_POP_4 412 +#define _GUARD_BIT_IS_SET_POP_5 413 +#define _GUARD_BIT_IS_SET_POP_6 414 +#define _GUARD_BIT_IS_SET_POP_7 415 +#define _GUARD_BIT_IS_UNSET_POP 416 +#define _GUARD_BIT_IS_UNSET_POP_4 417 +#define _GUARD_BIT_IS_UNSET_POP_5 418 +#define _GUARD_BIT_IS_UNSET_POP_6 419 +#define _GUARD_BIT_IS_UNSET_POP_7 420 +#define _GUARD_CALLABLE_BUILTIN_FAST 421 +#define _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS 422 +#define _GUARD_CALLABLE_BUILTIN_O 423 +#define _GUARD_CALLABLE_ISINSTANCE 424 +#define _GUARD_CALLABLE_LEN 425 +#define _GUARD_CALLABLE_LIST_APPEND 426 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST 427 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 428 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS 429 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_O 430 +#define _GUARD_CALLABLE_STR_1 431 +#define _GUARD_CALLABLE_TUPLE_1 432 +#define _GUARD_CALLABLE_TYPE_1 433 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR 434 +#define _GUARD_CODE_VERSION_RETURN_VALUE 435 +#define _GUARD_CODE_VERSION_YIELD_VALUE 436 +#define _GUARD_CODE_VERSION__PUSH_FRAME 437 +#define _GUARD_DORV_NO_DICT 438 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 439 +#define _GUARD_GLOBALS_VERSION 440 +#define _GUARD_IP_RETURN_GENERATOR 441 +#define _GUARD_IP_RETURN_VALUE 442 +#define _GUARD_IP_YIELD_VALUE 443 +#define _GUARD_IP__PUSH_FRAME 444 +#define _GUARD_IS_FALSE_POP 445 +#define _GUARD_IS_NONE_POP 446 +#define _GUARD_IS_NOT_NONE_POP 447 +#define _GUARD_IS_TRUE_POP 448 +#define _GUARD_KEYS_VERSION 449 +#define _GUARD_NOS_ANY_DICT 450 +#define _GUARD_NOS_COMPACT_ASCII 451 +#define _GUARD_NOS_DICT 452 +#define _GUARD_NOS_FLOAT 453 +#define _GUARD_NOS_INT 454 +#define _GUARD_NOS_LIST 455 +#define _GUARD_NOS_NOT_NULL 456 +#define _GUARD_NOS_NULL 457 +#define _GUARD_NOS_OVERFLOWED 458 +#define _GUARD_NOS_TUPLE 459 +#define _GUARD_NOS_UNICODE 460 +#define _GUARD_NOT_EXHAUSTED_LIST 461 +#define _GUARD_NOT_EXHAUSTED_RANGE 462 +#define _GUARD_NOT_EXHAUSTED_TUPLE 463 +#define _GUARD_THIRD_NULL 464 +#define _GUARD_TOS_ANY_DICT 465 +#define _GUARD_TOS_ANY_SET 466 +#define _GUARD_TOS_DICT 467 +#define _GUARD_TOS_FLOAT 468 +#define _GUARD_TOS_FROZENDICT 469 +#define _GUARD_TOS_FROZENSET 470 +#define _GUARD_TOS_INT 471 +#define _GUARD_TOS_LIST 472 +#define _GUARD_TOS_OVERFLOWED 473 +#define _GUARD_TOS_SET 474 +#define _GUARD_TOS_SLICE 475 +#define _GUARD_TOS_TUPLE 476 +#define _GUARD_TOS_UNICODE 477 +#define _GUARD_TYPE_VERSION 478 +#define _GUARD_TYPE_VERSION_LOCKED 479 +#define _HANDLE_PENDING_AND_DEOPT 480 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 453 -#define _INIT_CALL_PY_EXACT_ARGS 454 -#define _INIT_CALL_PY_EXACT_ARGS_0 455 -#define _INIT_CALL_PY_EXACT_ARGS_1 456 -#define _INIT_CALL_PY_EXACT_ARGS_2 457 -#define _INIT_CALL_PY_EXACT_ARGS_3 458 -#define _INIT_CALL_PY_EXACT_ARGS_4 459 -#define _INSERT_1_LOAD_CONST_INLINE 460 -#define _INSERT_1_LOAD_CONST_INLINE_BORROW 461 -#define _INSERT_2_LOAD_CONST_INLINE_BORROW 462 -#define _INSERT_NULL 463 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 481 +#define _INIT_CALL_PY_EXACT_ARGS 482 +#define _INIT_CALL_PY_EXACT_ARGS_0 483 +#define _INIT_CALL_PY_EXACT_ARGS_1 484 +#define _INIT_CALL_PY_EXACT_ARGS_2 485 +#define _INIT_CALL_PY_EXACT_ARGS_3 486 +#define _INIT_CALL_PY_EXACT_ARGS_4 487 +#define _INSERT_1_LOAD_CONST_INLINE 488 +#define _INSERT_1_LOAD_CONST_INLINE_BORROW 489 +#define _INSERT_2_LOAD_CONST_INLINE_BORROW 490 +#define _INSERT_NULL 491 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -216,1087 +240,1154 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 464 -#define _IS_OP 465 -#define _ITER_CHECK_LIST 466 -#define _ITER_CHECK_RANGE 467 -#define _ITER_CHECK_TUPLE 468 -#define _ITER_JUMP_LIST 469 -#define _ITER_JUMP_RANGE 470 -#define _ITER_JUMP_TUPLE 471 -#define _ITER_NEXT_LIST 472 -#define _ITER_NEXT_LIST_TIER_TWO 473 -#define _ITER_NEXT_RANGE 474 -#define _ITER_NEXT_TUPLE 475 +#define _IS_NONE 492 +#define _IS_OP 493 +#define _ITER_CHECK_LIST 494 +#define _ITER_CHECK_RANGE 495 +#define _ITER_CHECK_TUPLE 496 +#define _ITER_JUMP_LIST 497 +#define _ITER_JUMP_RANGE 498 +#define _ITER_JUMP_TUPLE 499 +#define _ITER_NEXT_LIST 500 +#define _ITER_NEXT_LIST_TIER_TWO 501 +#define _ITER_NEXT_RANGE 502 +#define _ITER_NEXT_TUPLE 503 #define _JUMP_BACKWARD_NO_INTERRUPT JUMP_BACKWARD_NO_INTERRUPT -#define _JUMP_TO_TOP 476 +#define _JUMP_TO_TOP 504 #define _LIST_APPEND LIST_APPEND -#define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 477 -#define _LOAD_ATTR_CLASS 478 +#define _LIST_EXTEND 505 +#define _LOAD_ATTR 506 +#define _LOAD_ATTR_CLASS 507 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 479 -#define _LOAD_ATTR_METHOD_LAZY_DICT 480 -#define _LOAD_ATTR_METHOD_NO_DICT 481 -#define _LOAD_ATTR_METHOD_WITH_VALUES 482 -#define _LOAD_ATTR_MODULE 483 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 484 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 485 -#define _LOAD_ATTR_PROPERTY_FRAME 486 -#define _LOAD_ATTR_SLOT 487 -#define _LOAD_ATTR_WITH_HINT 488 +#define _LOAD_ATTR_INSTANCE_VALUE 508 +#define _LOAD_ATTR_METHOD_LAZY_DICT 509 +#define _LOAD_ATTR_METHOD_NO_DICT 510 +#define _LOAD_ATTR_METHOD_WITH_VALUES 511 +#define _LOAD_ATTR_MODULE 512 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 513 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 514 +#define _LOAD_ATTR_PROPERTY_FRAME 515 +#define _LOAD_ATTR_SLOT 516 +#define _LOAD_ATTR_WITH_HINT 517 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 489 +#define _LOAD_BYTECODE 518 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 490 -#define _LOAD_CONST_INLINE_BORROW 491 -#define _LOAD_CONST_UNDER_INLINE 492 -#define _LOAD_CONST_UNDER_INLINE_BORROW 493 +#define _LOAD_CONST_INLINE 519 +#define _LOAD_CONST_INLINE_BORROW 520 +#define _LOAD_CONST_UNDER_INLINE 521 +#define _LOAD_CONST_UNDER_INLINE_BORROW 522 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 494 -#define _LOAD_FAST_0 495 -#define _LOAD_FAST_1 496 -#define _LOAD_FAST_2 497 -#define _LOAD_FAST_3 498 -#define _LOAD_FAST_4 499 -#define _LOAD_FAST_5 500 -#define _LOAD_FAST_6 501 -#define _LOAD_FAST_7 502 +#define _LOAD_FAST 523 +#define _LOAD_FAST_0 524 +#define _LOAD_FAST_1 525 +#define _LOAD_FAST_2 526 +#define _LOAD_FAST_3 527 +#define _LOAD_FAST_4 528 +#define _LOAD_FAST_5 529 +#define _LOAD_FAST_6 530 +#define _LOAD_FAST_7 531 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 503 -#define _LOAD_FAST_BORROW_0 504 -#define _LOAD_FAST_BORROW_1 505 -#define _LOAD_FAST_BORROW_2 506 -#define _LOAD_FAST_BORROW_3 507 -#define _LOAD_FAST_BORROW_4 508 -#define _LOAD_FAST_BORROW_5 509 -#define _LOAD_FAST_BORROW_6 510 -#define _LOAD_FAST_BORROW_7 511 +#define _LOAD_FAST_BORROW 532 +#define _LOAD_FAST_BORROW_0 533 +#define _LOAD_FAST_BORROW_1 534 +#define _LOAD_FAST_BORROW_2 535 +#define _LOAD_FAST_BORROW_3 536 +#define _LOAD_FAST_BORROW_4 537 +#define _LOAD_FAST_BORROW_5 538 +#define _LOAD_FAST_BORROW_6 539 +#define _LOAD_FAST_BORROW_7 540 #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 512 -#define _LOAD_GLOBAL_BUILTINS 513 -#define _LOAD_GLOBAL_MODULE 514 +#define _LOAD_GLOBAL 541 +#define _LOAD_GLOBAL_BUILTINS 542 +#define _LOAD_GLOBAL_MODULE 543 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 515 -#define _LOAD_SMALL_INT_0 516 -#define _LOAD_SMALL_INT_1 517 -#define _LOAD_SMALL_INT_2 518 -#define _LOAD_SMALL_INT_3 519 -#define _LOAD_SPECIAL 520 +#define _LOAD_SMALL_INT 544 +#define _LOAD_SMALL_INT_0 545 +#define _LOAD_SMALL_INT_1 546 +#define _LOAD_SMALL_INT_2 547 +#define _LOAD_SMALL_INT_3 548 +#define _LOAD_SPECIAL 549 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _LOCK_OBJECT 521 -#define _MAKE_CALLARGS_A_TUPLE 522 +#define _LOCK_OBJECT 550 +#define _MAKE_CALLARGS_A_TUPLE 551 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_HEAP_SAFE 523 -#define _MAKE_WARM 524 +#define _MAKE_HEAP_SAFE 552 +#define _MAKE_WARM 553 #define _MAP_ADD MAP_ADD -#define _MATCH_CLASS 525 +#define _MATCH_CLASS 554 #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 526 -#define _MAYBE_EXPAND_METHOD_KW 527 -#define _MONITOR_CALL 528 -#define _MONITOR_CALL_KW 529 -#define _MONITOR_JUMP_BACKWARD 530 -#define _MONITOR_RESUME 531 +#define _MAYBE_EXPAND_METHOD 555 +#define _MAYBE_EXPAND_METHOD_KW 556 +#define _MONITOR_CALL 557 +#define _MONITOR_CALL_KW 558 +#define _MONITOR_JUMP_BACKWARD 559 +#define _MONITOR_RESUME 560 #define _NOP NOP -#define _POP_CALL 532 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 533 -#define _POP_CALL_ONE 534 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 535 -#define _POP_CALL_TWO 536 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 537 +#define _POP_CALL 561 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 562 +#define _POP_CALL_ONE 563 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 564 +#define _POP_CALL_TWO 565 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 566 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 538 -#define _POP_JUMP_IF_TRUE 539 +#define _POP_JUMP_IF_FALSE 567 +#define _POP_JUMP_IF_TRUE 568 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 540 -#define _POP_TOP_INT 541 -#define _POP_TOP_LOAD_CONST_INLINE 542 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 543 -#define _POP_TOP_NOP 544 -#define _POP_TOP_UNICODE 545 -#define _POP_TWO 546 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 547 +#define _POP_TOP_FLOAT 569 +#define _POP_TOP_INT 570 +#define _POP_TOP_LOAD_CONST_INLINE 571 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 572 +#define _POP_TOP_NOP 573 +#define _POP_TOP_UNICODE 574 +#define _POP_TWO 575 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 576 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 548 +#define _PUSH_FRAME 577 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 549 -#define _PY_FRAME_EX 550 -#define _PY_FRAME_GENERAL 551 -#define _PY_FRAME_KW 552 -#define _RECORD_4OS 553 -#define _RECORD_BOUND_METHOD 554 -#define _RECORD_CALLABLE 555 -#define _RECORD_CODE 556 -#define _RECORD_NOS 557 -#define _RECORD_NOS_GEN_FUNC 558 -#define _RECORD_TOS 559 -#define _RECORD_TOS_TYPE 560 -#define _REPLACE_WITH_TRUE 561 -#define _RESUME_CHECK 562 +#define _PUSH_NULL_CONDITIONAL 578 +#define _PY_FRAME_EX 579 +#define _PY_FRAME_GENERAL 580 +#define _PY_FRAME_KW 581 +#define _RECORD_3OS_GEN_FUNC 582 +#define _RECORD_4OS 583 +#define _RECORD_BOUND_METHOD 584 +#define _RECORD_CALLABLE 585 +#define _RECORD_CODE 586 +#define _RECORD_NOS 587 +#define _RECORD_NOS_GEN_FUNC 588 +#define _RECORD_TOS 589 +#define _RECORD_TOS_TYPE 590 +#define _REPLACE_WITH_TRUE 591 +#define _RESUME_CHECK 592 #define _RETURN_GENERATOR RETURN_GENERATOR -#define _RETURN_VALUE 563 -#define _SAVE_RETURN_OFFSET 564 -#define _SEND 565 -#define _SEND_GEN_FRAME 566 +#define _RETURN_VALUE 593 +#define _SAVE_RETURN_OFFSET 594 +#define _SEND 595 +#define _SEND_GEN_FRAME 596 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE -#define _SET_UPDATE 567 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW 568 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW 569 -#define _SPILL_OR_RELOAD 570 -#define _START_EXECUTOR 571 -#define _STORE_ATTR 572 -#define _STORE_ATTR_INSTANCE_VALUE 573 -#define _STORE_ATTR_SLOT 574 -#define _STORE_ATTR_WITH_HINT 575 +#define _SET_UPDATE 597 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW 598 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW 599 +#define _SPILL_OR_RELOAD 600 +#define _START_EXECUTOR 601 +#define _STORE_ATTR 602 +#define _STORE_ATTR_INSTANCE_VALUE 603 +#define _STORE_ATTR_SLOT 604 +#define _STORE_ATTR_WITH_HINT 605 #define _STORE_DEREF STORE_DEREF #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 576 -#define _STORE_SUBSCR 577 -#define _STORE_SUBSCR_DICT 578 -#define _STORE_SUBSCR_LIST_INT 579 -#define _SWAP 580 -#define _SWAP_2 581 -#define _SWAP_3 582 -#define _SWAP_FAST 583 -#define _SWAP_FAST_0 584 -#define _SWAP_FAST_1 585 -#define _SWAP_FAST_2 586 -#define _SWAP_FAST_3 587 -#define _SWAP_FAST_4 588 -#define _SWAP_FAST_5 589 -#define _SWAP_FAST_6 590 -#define _SWAP_FAST_7 591 -#define _TIER2_RESUME_CHECK 592 -#define _TO_BOOL 593 +#define _STORE_SLICE 606 +#define _STORE_SUBSCR 607 +#define _STORE_SUBSCR_DICT 608 +#define _STORE_SUBSCR_DICT_KNOWN_HASH 609 +#define _STORE_SUBSCR_LIST_INT 610 +#define _SWAP 611 +#define _SWAP_2 612 +#define _SWAP_3 613 +#define _SWAP_FAST 614 +#define _SWAP_FAST_0 615 +#define _SWAP_FAST_1 616 +#define _SWAP_FAST_2 617 +#define _SWAP_FAST_3 618 +#define _SWAP_FAST_4 619 +#define _SWAP_FAST_5 620 +#define _SWAP_FAST_6 621 +#define _SWAP_FAST_7 622 +#define _TIER2_RESUME_CHECK 623 +#define _TO_BOOL 624 #define _TO_BOOL_BOOL TO_BOOL_BOOL -#define _TO_BOOL_INT 594 -#define _TO_BOOL_LIST 595 +#define _TO_BOOL_INT 625 +#define _TO_BOOL_LIST 626 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 596 +#define _TO_BOOL_STR 627 #define _TRACE_RECORD TRACE_RECORD -#define _UNARY_INVERT 597 -#define _UNARY_NEGATIVE 598 +#define _UNARY_INVERT 628 +#define _UNARY_NEGATIVE 629 +#define _UNARY_NEGATIVE_FLOAT_INPLACE 630 #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 599 -#define _UNPACK_SEQUENCE_LIST 600 -#define _UNPACK_SEQUENCE_TUPLE 601 -#define _UNPACK_SEQUENCE_TWO_TUPLE 602 +#define _UNPACK_SEQUENCE 631 +#define _UNPACK_SEQUENCE_LIST 632 +#define _UNPACK_SEQUENCE_TUPLE 633 +#define _UNPACK_SEQUENCE_TWO_TUPLE 634 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE 635 +#define _UNPACK_SEQUENCE_UNIQUE_TUPLE 636 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE 637 #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define _YIELD_VALUE 603 -#define MAX_UOP_ID 603 -#define _BINARY_OP_r23 604 -#define _BINARY_OP_ADD_FLOAT_r03 605 -#define _BINARY_OP_ADD_FLOAT_r13 606 -#define _BINARY_OP_ADD_FLOAT_r23 607 -#define _BINARY_OP_ADD_INT_r03 608 -#define _BINARY_OP_ADD_INT_r13 609 -#define _BINARY_OP_ADD_INT_r23 610 -#define _BINARY_OP_ADD_UNICODE_r03 611 -#define _BINARY_OP_ADD_UNICODE_r13 612 -#define _BINARY_OP_ADD_UNICODE_r23 613 -#define _BINARY_OP_EXTEND_r23 614 -#define _BINARY_OP_INPLACE_ADD_UNICODE_r21 615 -#define _BINARY_OP_MULTIPLY_FLOAT_r03 616 -#define _BINARY_OP_MULTIPLY_FLOAT_r13 617 -#define _BINARY_OP_MULTIPLY_FLOAT_r23 618 -#define _BINARY_OP_MULTIPLY_INT_r03 619 -#define _BINARY_OP_MULTIPLY_INT_r13 620 -#define _BINARY_OP_MULTIPLY_INT_r23 621 -#define _BINARY_OP_SUBSCR_CHECK_FUNC_r23 622 -#define _BINARY_OP_SUBSCR_DICT_r23 623 -#define _BINARY_OP_SUBSCR_INIT_CALL_r01 624 -#define _BINARY_OP_SUBSCR_INIT_CALL_r11 625 -#define _BINARY_OP_SUBSCR_INIT_CALL_r21 626 -#define _BINARY_OP_SUBSCR_INIT_CALL_r31 627 -#define _BINARY_OP_SUBSCR_LIST_INT_r23 628 -#define _BINARY_OP_SUBSCR_LIST_SLICE_r23 629 -#define _BINARY_OP_SUBSCR_STR_INT_r23 630 -#define _BINARY_OP_SUBSCR_TUPLE_INT_r03 631 -#define _BINARY_OP_SUBSCR_TUPLE_INT_r13 632 -#define _BINARY_OP_SUBSCR_TUPLE_INT_r23 633 -#define _BINARY_OP_SUBSCR_USTR_INT_r23 634 -#define _BINARY_OP_SUBTRACT_FLOAT_r03 635 -#define _BINARY_OP_SUBTRACT_FLOAT_r13 636 -#define _BINARY_OP_SUBTRACT_FLOAT_r23 637 -#define _BINARY_OP_SUBTRACT_INT_r03 638 -#define _BINARY_OP_SUBTRACT_INT_r13 639 -#define _BINARY_OP_SUBTRACT_INT_r23 640 -#define _BINARY_SLICE_r31 641 -#define _BUILD_INTERPOLATION_r01 642 -#define _BUILD_LIST_r01 643 -#define _BUILD_MAP_r01 644 -#define _BUILD_SET_r01 645 -#define _BUILD_SLICE_r01 646 -#define _BUILD_STRING_r01 647 -#define _BUILD_TEMPLATE_r21 648 -#define _BUILD_TUPLE_r01 649 -#define _CALL_BUILTIN_CLASS_r01 650 -#define _CALL_BUILTIN_FAST_r01 651 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS_r01 652 -#define _CALL_BUILTIN_O_r03 653 -#define _CALL_FUNCTION_EX_NON_PY_GENERAL_r31 654 -#define _CALL_INTRINSIC_1_r12 655 -#define _CALL_INTRINSIC_2_r21 656 -#define _CALL_ISINSTANCE_r31 657 -#define _CALL_KW_NON_PY_r11 658 -#define _CALL_LEN_r33 659 -#define _CALL_LIST_APPEND_r03 660 -#define _CALL_LIST_APPEND_r13 661 -#define _CALL_LIST_APPEND_r23 662 -#define _CALL_LIST_APPEND_r33 663 -#define _CALL_METHOD_DESCRIPTOR_FAST_r01 664 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01 665 -#define _CALL_METHOD_DESCRIPTOR_NOARGS_r01 666 -#define _CALL_METHOD_DESCRIPTOR_O_r03 667 -#define _CALL_NON_PY_GENERAL_r01 668 -#define _CALL_STR_1_r32 669 -#define _CALL_TUPLE_1_r32 670 -#define _CALL_TYPE_1_r02 671 -#define _CALL_TYPE_1_r12 672 -#define _CALL_TYPE_1_r22 673 -#define _CALL_TYPE_1_r32 674 -#define _CHECK_AND_ALLOCATE_OBJECT_r00 675 -#define _CHECK_ATTR_CLASS_r01 676 -#define _CHECK_ATTR_CLASS_r11 677 -#define _CHECK_ATTR_CLASS_r22 678 -#define _CHECK_ATTR_CLASS_r33 679 -#define _CHECK_ATTR_METHOD_LAZY_DICT_r01 680 -#define _CHECK_ATTR_METHOD_LAZY_DICT_r11 681 -#define _CHECK_ATTR_METHOD_LAZY_DICT_r22 682 -#define _CHECK_ATTR_METHOD_LAZY_DICT_r33 683 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00 684 -#define _CHECK_EG_MATCH_r22 685 -#define _CHECK_EXC_MATCH_r22 686 -#define _CHECK_FUNCTION_EXACT_ARGS_r00 687 -#define _CHECK_FUNCTION_VERSION_r00 688 -#define _CHECK_FUNCTION_VERSION_INLINE_r00 689 -#define _CHECK_FUNCTION_VERSION_INLINE_r11 690 -#define _CHECK_FUNCTION_VERSION_INLINE_r22 691 -#define _CHECK_FUNCTION_VERSION_INLINE_r33 692 -#define _CHECK_FUNCTION_VERSION_KW_r11 693 -#define _CHECK_IS_NOT_PY_CALLABLE_r00 694 -#define _CHECK_IS_NOT_PY_CALLABLE_EX_r03 695 -#define _CHECK_IS_NOT_PY_CALLABLE_EX_r13 696 -#define _CHECK_IS_NOT_PY_CALLABLE_EX_r23 697 -#define _CHECK_IS_NOT_PY_CALLABLE_EX_r33 698 -#define _CHECK_IS_NOT_PY_CALLABLE_KW_r11 699 -#define _CHECK_IS_PY_CALLABLE_EX_r03 700 -#define _CHECK_IS_PY_CALLABLE_EX_r13 701 -#define _CHECK_IS_PY_CALLABLE_EX_r23 702 -#define _CHECK_IS_PY_CALLABLE_EX_r33 703 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r01 704 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r11 705 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r22 706 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r33 707 -#define _CHECK_METHOD_VERSION_r00 708 -#define _CHECK_METHOD_VERSION_KW_r11 709 -#define _CHECK_PEP_523_r00 710 -#define _CHECK_PEP_523_r11 711 -#define _CHECK_PEP_523_r22 712 -#define _CHECK_PEP_523_r33 713 -#define _CHECK_PERIODIC_r00 714 -#define _CHECK_PERIODIC_AT_END_r00 715 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00 716 -#define _CHECK_RECURSION_REMAINING_r00 717 -#define _CHECK_RECURSION_REMAINING_r11 718 -#define _CHECK_RECURSION_REMAINING_r22 719 -#define _CHECK_RECURSION_REMAINING_r33 720 -#define _CHECK_STACK_SPACE_r00 721 -#define _CHECK_STACK_SPACE_OPERAND_r00 722 -#define _CHECK_STACK_SPACE_OPERAND_r11 723 -#define _CHECK_STACK_SPACE_OPERAND_r22 724 -#define _CHECK_STACK_SPACE_OPERAND_r33 725 -#define _CHECK_VALIDITY_r00 726 -#define _CHECK_VALIDITY_r11 727 -#define _CHECK_VALIDITY_r22 728 -#define _CHECK_VALIDITY_r33 729 -#define _COLD_DYNAMIC_EXIT_r00 730 -#define _COLD_EXIT_r00 731 -#define _COMPARE_OP_r21 732 -#define _COMPARE_OP_FLOAT_r03 733 -#define _COMPARE_OP_FLOAT_r13 734 -#define _COMPARE_OP_FLOAT_r23 735 -#define _COMPARE_OP_INT_r23 736 -#define _COMPARE_OP_STR_r23 737 -#define _CONTAINS_OP_r23 738 -#define _CONTAINS_OP_DICT_r23 739 -#define _CONTAINS_OP_SET_r23 740 -#define _CONVERT_VALUE_r11 741 -#define _COPY_r01 742 -#define _COPY_1_r02 743 -#define _COPY_1_r12 744 -#define _COPY_1_r23 745 -#define _COPY_2_r03 746 -#define _COPY_2_r13 747 -#define _COPY_2_r23 748 -#define _COPY_3_r03 749 -#define _COPY_3_r13 750 -#define _COPY_3_r23 751 -#define _COPY_3_r33 752 -#define _COPY_FREE_VARS_r00 753 -#define _COPY_FREE_VARS_r11 754 -#define _COPY_FREE_VARS_r22 755 -#define _COPY_FREE_VARS_r33 756 -#define _CREATE_INIT_FRAME_r01 757 -#define _DELETE_ATTR_r10 758 -#define _DELETE_DEREF_r00 759 -#define _DELETE_FAST_r00 760 -#define _DELETE_GLOBAL_r00 761 -#define _DELETE_NAME_r00 762 -#define _DELETE_SUBSCR_r20 763 -#define _DEOPT_r00 764 -#define _DEOPT_r10 765 -#define _DEOPT_r20 766 -#define _DEOPT_r30 767 -#define _DICT_MERGE_r10 768 -#define _DICT_UPDATE_r10 769 -#define _DO_CALL_r01 770 -#define _DO_CALL_FUNCTION_EX_r31 771 -#define _DO_CALL_KW_r11 772 -#define _DYNAMIC_EXIT_r00 773 -#define _DYNAMIC_EXIT_r10 774 -#define _DYNAMIC_EXIT_r20 775 -#define _DYNAMIC_EXIT_r30 776 -#define _END_FOR_r10 777 -#define _END_SEND_r21 778 -#define _ERROR_POP_N_r00 779 -#define _EXIT_INIT_CHECK_r10 780 -#define _EXIT_TRACE_r00 781 -#define _EXIT_TRACE_r10 782 -#define _EXIT_TRACE_r20 783 -#define _EXIT_TRACE_r30 784 -#define _EXPAND_METHOD_r00 785 -#define _EXPAND_METHOD_KW_r11 786 -#define _FATAL_ERROR_r00 787 -#define _FATAL_ERROR_r11 788 -#define _FATAL_ERROR_r22 789 -#define _FATAL_ERROR_r33 790 -#define _FORMAT_SIMPLE_r11 791 -#define _FORMAT_WITH_SPEC_r21 792 -#define _FOR_ITER_r23 793 -#define _FOR_ITER_GEN_FRAME_r03 794 -#define _FOR_ITER_GEN_FRAME_r13 795 -#define _FOR_ITER_GEN_FRAME_r23 796 -#define _FOR_ITER_TIER_TWO_r23 797 -#define _GET_AITER_r11 798 -#define _GET_ANEXT_r12 799 -#define _GET_AWAITABLE_r11 800 -#define _GET_ITER_r12 801 -#define _GET_LEN_r12 802 -#define _GET_YIELD_FROM_ITER_r11 803 -#define _GUARD_BINARY_OP_EXTEND_r22 804 -#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02 805 -#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12 806 -#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22 807 -#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33 808 -#define _GUARD_BIT_IS_SET_POP_r00 809 -#define _GUARD_BIT_IS_SET_POP_r10 810 -#define _GUARD_BIT_IS_SET_POP_r21 811 -#define _GUARD_BIT_IS_SET_POP_r32 812 -#define _GUARD_BIT_IS_SET_POP_4_r00 813 -#define _GUARD_BIT_IS_SET_POP_4_r10 814 -#define _GUARD_BIT_IS_SET_POP_4_r21 815 -#define _GUARD_BIT_IS_SET_POP_4_r32 816 -#define _GUARD_BIT_IS_SET_POP_5_r00 817 -#define _GUARD_BIT_IS_SET_POP_5_r10 818 -#define _GUARD_BIT_IS_SET_POP_5_r21 819 -#define _GUARD_BIT_IS_SET_POP_5_r32 820 -#define _GUARD_BIT_IS_SET_POP_6_r00 821 -#define _GUARD_BIT_IS_SET_POP_6_r10 822 -#define _GUARD_BIT_IS_SET_POP_6_r21 823 -#define _GUARD_BIT_IS_SET_POP_6_r32 824 -#define _GUARD_BIT_IS_SET_POP_7_r00 825 -#define _GUARD_BIT_IS_SET_POP_7_r10 826 -#define _GUARD_BIT_IS_SET_POP_7_r21 827 -#define _GUARD_BIT_IS_SET_POP_7_r32 828 -#define _GUARD_BIT_IS_UNSET_POP_r00 829 -#define _GUARD_BIT_IS_UNSET_POP_r10 830 -#define _GUARD_BIT_IS_UNSET_POP_r21 831 -#define _GUARD_BIT_IS_UNSET_POP_r32 832 -#define _GUARD_BIT_IS_UNSET_POP_4_r00 833 -#define _GUARD_BIT_IS_UNSET_POP_4_r10 834 -#define _GUARD_BIT_IS_UNSET_POP_4_r21 835 -#define _GUARD_BIT_IS_UNSET_POP_4_r32 836 -#define _GUARD_BIT_IS_UNSET_POP_5_r00 837 -#define _GUARD_BIT_IS_UNSET_POP_5_r10 838 -#define _GUARD_BIT_IS_UNSET_POP_5_r21 839 -#define _GUARD_BIT_IS_UNSET_POP_5_r32 840 -#define _GUARD_BIT_IS_UNSET_POP_6_r00 841 -#define _GUARD_BIT_IS_UNSET_POP_6_r10 842 -#define _GUARD_BIT_IS_UNSET_POP_6_r21 843 -#define _GUARD_BIT_IS_UNSET_POP_6_r32 844 -#define _GUARD_BIT_IS_UNSET_POP_7_r00 845 -#define _GUARD_BIT_IS_UNSET_POP_7_r10 846 -#define _GUARD_BIT_IS_UNSET_POP_7_r21 847 -#define _GUARD_BIT_IS_UNSET_POP_7_r32 848 -#define _GUARD_CALLABLE_ISINSTANCE_r03 849 -#define _GUARD_CALLABLE_ISINSTANCE_r13 850 -#define _GUARD_CALLABLE_ISINSTANCE_r23 851 -#define _GUARD_CALLABLE_ISINSTANCE_r33 852 -#define _GUARD_CALLABLE_LEN_r03 853 -#define _GUARD_CALLABLE_LEN_r13 854 -#define _GUARD_CALLABLE_LEN_r23 855 -#define _GUARD_CALLABLE_LEN_r33 856 -#define _GUARD_CALLABLE_LIST_APPEND_r03 857 -#define _GUARD_CALLABLE_LIST_APPEND_r13 858 -#define _GUARD_CALLABLE_LIST_APPEND_r23 859 -#define _GUARD_CALLABLE_LIST_APPEND_r33 860 -#define _GUARD_CALLABLE_STR_1_r03 861 -#define _GUARD_CALLABLE_STR_1_r13 862 -#define _GUARD_CALLABLE_STR_1_r23 863 -#define _GUARD_CALLABLE_STR_1_r33 864 -#define _GUARD_CALLABLE_TUPLE_1_r03 865 -#define _GUARD_CALLABLE_TUPLE_1_r13 866 -#define _GUARD_CALLABLE_TUPLE_1_r23 867 -#define _GUARD_CALLABLE_TUPLE_1_r33 868 -#define _GUARD_CALLABLE_TYPE_1_r03 869 -#define _GUARD_CALLABLE_TYPE_1_r13 870 -#define _GUARD_CALLABLE_TYPE_1_r23 871 -#define _GUARD_CALLABLE_TYPE_1_r33 872 -#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r00 873 -#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r11 874 -#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r22 875 -#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r33 876 -#define _GUARD_CODE_VERSION_RETURN_VALUE_r00 877 -#define _GUARD_CODE_VERSION_RETURN_VALUE_r11 878 -#define _GUARD_CODE_VERSION_RETURN_VALUE_r22 879 -#define _GUARD_CODE_VERSION_RETURN_VALUE_r33 880 -#define _GUARD_CODE_VERSION_YIELD_VALUE_r00 881 -#define _GUARD_CODE_VERSION_YIELD_VALUE_r11 882 -#define _GUARD_CODE_VERSION_YIELD_VALUE_r22 883 -#define _GUARD_CODE_VERSION_YIELD_VALUE_r33 884 -#define _GUARD_CODE_VERSION__PUSH_FRAME_r00 885 -#define _GUARD_CODE_VERSION__PUSH_FRAME_r11 886 -#define _GUARD_CODE_VERSION__PUSH_FRAME_r22 887 -#define _GUARD_CODE_VERSION__PUSH_FRAME_r33 888 -#define _GUARD_DORV_NO_DICT_r01 889 -#define _GUARD_DORV_NO_DICT_r11 890 -#define _GUARD_DORV_NO_DICT_r22 891 -#define _GUARD_DORV_NO_DICT_r33 892 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01 893 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 894 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 895 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 896 -#define _GUARD_GLOBALS_VERSION_r00 897 -#define _GUARD_GLOBALS_VERSION_r11 898 -#define _GUARD_GLOBALS_VERSION_r22 899 -#define _GUARD_GLOBALS_VERSION_r33 900 -#define _GUARD_IP_RETURN_GENERATOR_r00 901 -#define _GUARD_IP_RETURN_GENERATOR_r11 902 -#define _GUARD_IP_RETURN_GENERATOR_r22 903 -#define _GUARD_IP_RETURN_GENERATOR_r33 904 -#define _GUARD_IP_RETURN_VALUE_r00 905 -#define _GUARD_IP_RETURN_VALUE_r11 906 -#define _GUARD_IP_RETURN_VALUE_r22 907 -#define _GUARD_IP_RETURN_VALUE_r33 908 -#define _GUARD_IP_YIELD_VALUE_r00 909 -#define _GUARD_IP_YIELD_VALUE_r11 910 -#define _GUARD_IP_YIELD_VALUE_r22 911 -#define _GUARD_IP_YIELD_VALUE_r33 912 -#define _GUARD_IP__PUSH_FRAME_r00 913 -#define _GUARD_IP__PUSH_FRAME_r11 914 -#define _GUARD_IP__PUSH_FRAME_r22 915 -#define _GUARD_IP__PUSH_FRAME_r33 916 -#define _GUARD_IS_FALSE_POP_r00 917 -#define _GUARD_IS_FALSE_POP_r10 918 -#define _GUARD_IS_FALSE_POP_r21 919 -#define _GUARD_IS_FALSE_POP_r32 920 -#define _GUARD_IS_NONE_POP_r00 921 -#define _GUARD_IS_NONE_POP_r10 922 -#define _GUARD_IS_NONE_POP_r21 923 -#define _GUARD_IS_NONE_POP_r32 924 -#define _GUARD_IS_NOT_NONE_POP_r10 925 -#define _GUARD_IS_TRUE_POP_r00 926 -#define _GUARD_IS_TRUE_POP_r10 927 -#define _GUARD_IS_TRUE_POP_r21 928 -#define _GUARD_IS_TRUE_POP_r32 929 -#define _GUARD_KEYS_VERSION_r01 930 -#define _GUARD_KEYS_VERSION_r11 931 -#define _GUARD_KEYS_VERSION_r22 932 -#define _GUARD_KEYS_VERSION_r33 933 -#define _GUARD_NOS_ANY_DICT_r02 934 -#define _GUARD_NOS_ANY_DICT_r12 935 -#define _GUARD_NOS_ANY_DICT_r22 936 -#define _GUARD_NOS_ANY_DICT_r33 937 -#define _GUARD_NOS_COMPACT_ASCII_r02 938 -#define _GUARD_NOS_COMPACT_ASCII_r12 939 -#define _GUARD_NOS_COMPACT_ASCII_r22 940 -#define _GUARD_NOS_COMPACT_ASCII_r33 941 -#define _GUARD_NOS_DICT_r02 942 -#define _GUARD_NOS_DICT_r12 943 -#define _GUARD_NOS_DICT_r22 944 -#define _GUARD_NOS_DICT_r33 945 -#define _GUARD_NOS_FLOAT_r02 946 -#define _GUARD_NOS_FLOAT_r12 947 -#define _GUARD_NOS_FLOAT_r22 948 -#define _GUARD_NOS_FLOAT_r33 949 -#define _GUARD_NOS_INT_r02 950 -#define _GUARD_NOS_INT_r12 951 -#define _GUARD_NOS_INT_r22 952 -#define _GUARD_NOS_INT_r33 953 -#define _GUARD_NOS_LIST_r02 954 -#define _GUARD_NOS_LIST_r12 955 -#define _GUARD_NOS_LIST_r22 956 -#define _GUARD_NOS_LIST_r33 957 -#define _GUARD_NOS_NOT_NULL_r02 958 -#define _GUARD_NOS_NOT_NULL_r12 959 -#define _GUARD_NOS_NOT_NULL_r22 960 -#define _GUARD_NOS_NOT_NULL_r33 961 -#define _GUARD_NOS_NULL_r02 962 -#define _GUARD_NOS_NULL_r12 963 -#define _GUARD_NOS_NULL_r22 964 -#define _GUARD_NOS_NULL_r33 965 -#define _GUARD_NOS_OVERFLOWED_r02 966 -#define _GUARD_NOS_OVERFLOWED_r12 967 -#define _GUARD_NOS_OVERFLOWED_r22 968 -#define _GUARD_NOS_OVERFLOWED_r33 969 -#define _GUARD_NOS_TUPLE_r02 970 -#define _GUARD_NOS_TUPLE_r12 971 -#define _GUARD_NOS_TUPLE_r22 972 -#define _GUARD_NOS_TUPLE_r33 973 -#define _GUARD_NOS_UNICODE_r02 974 -#define _GUARD_NOS_UNICODE_r12 975 -#define _GUARD_NOS_UNICODE_r22 976 -#define _GUARD_NOS_UNICODE_r33 977 -#define _GUARD_NOT_EXHAUSTED_LIST_r02 978 -#define _GUARD_NOT_EXHAUSTED_LIST_r12 979 -#define _GUARD_NOT_EXHAUSTED_LIST_r22 980 -#define _GUARD_NOT_EXHAUSTED_LIST_r33 981 -#define _GUARD_NOT_EXHAUSTED_RANGE_r02 982 -#define _GUARD_NOT_EXHAUSTED_RANGE_r12 983 -#define _GUARD_NOT_EXHAUSTED_RANGE_r22 984 -#define _GUARD_NOT_EXHAUSTED_RANGE_r33 985 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r02 986 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r12 987 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r22 988 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r33 989 -#define _GUARD_THIRD_NULL_r03 990 -#define _GUARD_THIRD_NULL_r13 991 -#define _GUARD_THIRD_NULL_r23 992 -#define _GUARD_THIRD_NULL_r33 993 -#define _GUARD_TOS_ANY_DICT_r01 994 -#define _GUARD_TOS_ANY_DICT_r11 995 -#define _GUARD_TOS_ANY_DICT_r22 996 -#define _GUARD_TOS_ANY_DICT_r33 997 -#define _GUARD_TOS_ANY_SET_r01 998 -#define _GUARD_TOS_ANY_SET_r11 999 -#define _GUARD_TOS_ANY_SET_r22 1000 -#define _GUARD_TOS_ANY_SET_r33 1001 -#define _GUARD_TOS_DICT_r01 1002 -#define _GUARD_TOS_DICT_r11 1003 -#define _GUARD_TOS_DICT_r22 1004 -#define _GUARD_TOS_DICT_r33 1005 -#define _GUARD_TOS_FLOAT_r01 1006 -#define _GUARD_TOS_FLOAT_r11 1007 -#define _GUARD_TOS_FLOAT_r22 1008 -#define _GUARD_TOS_FLOAT_r33 1009 -#define _GUARD_TOS_FROZENDICT_r01 1010 -#define _GUARD_TOS_FROZENDICT_r11 1011 -#define _GUARD_TOS_FROZENDICT_r22 1012 -#define _GUARD_TOS_FROZENDICT_r33 1013 -#define _GUARD_TOS_FROZENSET_r01 1014 -#define _GUARD_TOS_FROZENSET_r11 1015 -#define _GUARD_TOS_FROZENSET_r22 1016 -#define _GUARD_TOS_FROZENSET_r33 1017 -#define _GUARD_TOS_INT_r01 1018 -#define _GUARD_TOS_INT_r11 1019 -#define _GUARD_TOS_INT_r22 1020 -#define _GUARD_TOS_INT_r33 1021 -#define _GUARD_TOS_LIST_r01 1022 -#define _GUARD_TOS_LIST_r11 1023 -#define _GUARD_TOS_LIST_r22 1024 -#define _GUARD_TOS_LIST_r33 1025 -#define _GUARD_TOS_OVERFLOWED_r01 1026 -#define _GUARD_TOS_OVERFLOWED_r11 1027 -#define _GUARD_TOS_OVERFLOWED_r22 1028 -#define _GUARD_TOS_OVERFLOWED_r33 1029 -#define _GUARD_TOS_SET_r01 1030 -#define _GUARD_TOS_SET_r11 1031 -#define _GUARD_TOS_SET_r22 1032 -#define _GUARD_TOS_SET_r33 1033 -#define _GUARD_TOS_SLICE_r01 1034 -#define _GUARD_TOS_SLICE_r11 1035 -#define _GUARD_TOS_SLICE_r22 1036 -#define _GUARD_TOS_SLICE_r33 1037 -#define _GUARD_TOS_TUPLE_r01 1038 -#define _GUARD_TOS_TUPLE_r11 1039 -#define _GUARD_TOS_TUPLE_r22 1040 -#define _GUARD_TOS_TUPLE_r33 1041 -#define _GUARD_TOS_UNICODE_r01 1042 -#define _GUARD_TOS_UNICODE_r11 1043 -#define _GUARD_TOS_UNICODE_r22 1044 -#define _GUARD_TOS_UNICODE_r33 1045 -#define _GUARD_TYPE_VERSION_r01 1046 -#define _GUARD_TYPE_VERSION_r11 1047 -#define _GUARD_TYPE_VERSION_r22 1048 -#define _GUARD_TYPE_VERSION_r33 1049 -#define _GUARD_TYPE_VERSION_LOCKED_r01 1050 -#define _GUARD_TYPE_VERSION_LOCKED_r11 1051 -#define _GUARD_TYPE_VERSION_LOCKED_r22 1052 -#define _GUARD_TYPE_VERSION_LOCKED_r33 1053 -#define _HANDLE_PENDING_AND_DEOPT_r00 1054 -#define _HANDLE_PENDING_AND_DEOPT_r10 1055 -#define _HANDLE_PENDING_AND_DEOPT_r20 1056 -#define _HANDLE_PENDING_AND_DEOPT_r30 1057 -#define _IMPORT_FROM_r12 1058 -#define _IMPORT_NAME_r21 1059 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 1060 -#define _INIT_CALL_PY_EXACT_ARGS_r01 1061 -#define _INIT_CALL_PY_EXACT_ARGS_0_r01 1062 -#define _INIT_CALL_PY_EXACT_ARGS_1_r01 1063 -#define _INIT_CALL_PY_EXACT_ARGS_2_r01 1064 -#define _INIT_CALL_PY_EXACT_ARGS_3_r01 1065 -#define _INIT_CALL_PY_EXACT_ARGS_4_r01 1066 -#define _INSERT_1_LOAD_CONST_INLINE_r02 1067 -#define _INSERT_1_LOAD_CONST_INLINE_r12 1068 -#define _INSERT_1_LOAD_CONST_INLINE_r23 1069 -#define _INSERT_1_LOAD_CONST_INLINE_BORROW_r02 1070 -#define _INSERT_1_LOAD_CONST_INLINE_BORROW_r12 1071 -#define _INSERT_1_LOAD_CONST_INLINE_BORROW_r23 1072 -#define _INSERT_2_LOAD_CONST_INLINE_BORROW_r03 1073 -#define _INSERT_2_LOAD_CONST_INLINE_BORROW_r13 1074 -#define _INSERT_2_LOAD_CONST_INLINE_BORROW_r23 1075 -#define _INSERT_NULL_r10 1076 -#define _INSTRUMENTED_FOR_ITER_r23 1077 -#define _INSTRUMENTED_INSTRUCTION_r00 1078 -#define _INSTRUMENTED_JUMP_FORWARD_r00 1079 -#define _INSTRUMENTED_JUMP_FORWARD_r11 1080 -#define _INSTRUMENTED_JUMP_FORWARD_r22 1081 -#define _INSTRUMENTED_JUMP_FORWARD_r33 1082 -#define _INSTRUMENTED_LINE_r00 1083 -#define _INSTRUMENTED_NOT_TAKEN_r00 1084 -#define _INSTRUMENTED_NOT_TAKEN_r11 1085 -#define _INSTRUMENTED_NOT_TAKEN_r22 1086 -#define _INSTRUMENTED_NOT_TAKEN_r33 1087 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 1088 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 1089 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 1090 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 1091 -#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 1092 -#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 1093 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 1094 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 1095 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 1096 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 1097 -#define _IS_NONE_r11 1098 -#define _IS_OP_r03 1099 -#define _IS_OP_r13 1100 -#define _IS_OP_r23 1101 -#define _ITER_CHECK_LIST_r02 1102 -#define _ITER_CHECK_LIST_r12 1103 -#define _ITER_CHECK_LIST_r22 1104 -#define _ITER_CHECK_LIST_r33 1105 -#define _ITER_CHECK_RANGE_r02 1106 -#define _ITER_CHECK_RANGE_r12 1107 -#define _ITER_CHECK_RANGE_r22 1108 -#define _ITER_CHECK_RANGE_r33 1109 -#define _ITER_CHECK_TUPLE_r02 1110 -#define _ITER_CHECK_TUPLE_r12 1111 -#define _ITER_CHECK_TUPLE_r22 1112 -#define _ITER_CHECK_TUPLE_r33 1113 -#define _ITER_JUMP_LIST_r02 1114 -#define _ITER_JUMP_LIST_r12 1115 -#define _ITER_JUMP_LIST_r22 1116 -#define _ITER_JUMP_LIST_r33 1117 -#define _ITER_JUMP_RANGE_r02 1118 -#define _ITER_JUMP_RANGE_r12 1119 -#define _ITER_JUMP_RANGE_r22 1120 -#define _ITER_JUMP_RANGE_r33 1121 -#define _ITER_JUMP_TUPLE_r02 1122 -#define _ITER_JUMP_TUPLE_r12 1123 -#define _ITER_JUMP_TUPLE_r22 1124 -#define _ITER_JUMP_TUPLE_r33 1125 -#define _ITER_NEXT_LIST_r23 1126 -#define _ITER_NEXT_LIST_TIER_TWO_r23 1127 -#define _ITER_NEXT_RANGE_r03 1128 -#define _ITER_NEXT_RANGE_r13 1129 -#define _ITER_NEXT_RANGE_r23 1130 -#define _ITER_NEXT_TUPLE_r03 1131 -#define _ITER_NEXT_TUPLE_r13 1132 -#define _ITER_NEXT_TUPLE_r23 1133 -#define _JUMP_BACKWARD_NO_INTERRUPT_r00 1134 -#define _JUMP_BACKWARD_NO_INTERRUPT_r11 1135 -#define _JUMP_BACKWARD_NO_INTERRUPT_r22 1136 -#define _JUMP_BACKWARD_NO_INTERRUPT_r33 1137 -#define _JUMP_TO_TOP_r00 1138 -#define _LIST_APPEND_r10 1139 -#define _LIST_EXTEND_r10 1140 -#define _LOAD_ATTR_r10 1141 -#define _LOAD_ATTR_CLASS_r11 1142 -#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_r11 1143 -#define _LOAD_ATTR_INSTANCE_VALUE_r02 1144 -#define _LOAD_ATTR_INSTANCE_VALUE_r12 1145 -#define _LOAD_ATTR_INSTANCE_VALUE_r23 1146 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 1147 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 1148 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 1149 -#define _LOAD_ATTR_METHOD_NO_DICT_r02 1150 -#define _LOAD_ATTR_METHOD_NO_DICT_r12 1151 -#define _LOAD_ATTR_METHOD_NO_DICT_r23 1152 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 1153 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 1154 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1155 -#define _LOAD_ATTR_MODULE_r12 1156 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1157 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1158 -#define _LOAD_ATTR_PROPERTY_FRAME_r11 1159 -#define _LOAD_ATTR_SLOT_r02 1160 -#define _LOAD_ATTR_SLOT_r12 1161 -#define _LOAD_ATTR_SLOT_r23 1162 -#define _LOAD_ATTR_WITH_HINT_r12 1163 -#define _LOAD_BUILD_CLASS_r01 1164 -#define _LOAD_BYTECODE_r00 1165 -#define _LOAD_COMMON_CONSTANT_r01 1166 -#define _LOAD_COMMON_CONSTANT_r12 1167 -#define _LOAD_COMMON_CONSTANT_r23 1168 -#define _LOAD_CONST_r01 1169 -#define _LOAD_CONST_r12 1170 -#define _LOAD_CONST_r23 1171 -#define _LOAD_CONST_INLINE_r01 1172 -#define _LOAD_CONST_INLINE_r12 1173 -#define _LOAD_CONST_INLINE_r23 1174 -#define _LOAD_CONST_INLINE_BORROW_r01 1175 -#define _LOAD_CONST_INLINE_BORROW_r12 1176 -#define _LOAD_CONST_INLINE_BORROW_r23 1177 -#define _LOAD_CONST_UNDER_INLINE_r02 1178 -#define _LOAD_CONST_UNDER_INLINE_r12 1179 -#define _LOAD_CONST_UNDER_INLINE_r23 1180 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r02 1181 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r12 1182 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r23 1183 -#define _LOAD_DEREF_r01 1184 -#define _LOAD_FAST_r01 1185 -#define _LOAD_FAST_r12 1186 -#define _LOAD_FAST_r23 1187 -#define _LOAD_FAST_0_r01 1188 -#define _LOAD_FAST_0_r12 1189 -#define _LOAD_FAST_0_r23 1190 -#define _LOAD_FAST_1_r01 1191 -#define _LOAD_FAST_1_r12 1192 -#define _LOAD_FAST_1_r23 1193 -#define _LOAD_FAST_2_r01 1194 -#define _LOAD_FAST_2_r12 1195 -#define _LOAD_FAST_2_r23 1196 -#define _LOAD_FAST_3_r01 1197 -#define _LOAD_FAST_3_r12 1198 -#define _LOAD_FAST_3_r23 1199 -#define _LOAD_FAST_4_r01 1200 -#define _LOAD_FAST_4_r12 1201 -#define _LOAD_FAST_4_r23 1202 -#define _LOAD_FAST_5_r01 1203 -#define _LOAD_FAST_5_r12 1204 -#define _LOAD_FAST_5_r23 1205 -#define _LOAD_FAST_6_r01 1206 -#define _LOAD_FAST_6_r12 1207 -#define _LOAD_FAST_6_r23 1208 -#define _LOAD_FAST_7_r01 1209 -#define _LOAD_FAST_7_r12 1210 -#define _LOAD_FAST_7_r23 1211 -#define _LOAD_FAST_AND_CLEAR_r01 1212 -#define _LOAD_FAST_AND_CLEAR_r12 1213 -#define _LOAD_FAST_AND_CLEAR_r23 1214 -#define _LOAD_FAST_BORROW_r01 1215 -#define _LOAD_FAST_BORROW_r12 1216 -#define _LOAD_FAST_BORROW_r23 1217 -#define _LOAD_FAST_BORROW_0_r01 1218 -#define _LOAD_FAST_BORROW_0_r12 1219 -#define _LOAD_FAST_BORROW_0_r23 1220 -#define _LOAD_FAST_BORROW_1_r01 1221 -#define _LOAD_FAST_BORROW_1_r12 1222 -#define _LOAD_FAST_BORROW_1_r23 1223 -#define _LOAD_FAST_BORROW_2_r01 1224 -#define _LOAD_FAST_BORROW_2_r12 1225 -#define _LOAD_FAST_BORROW_2_r23 1226 -#define _LOAD_FAST_BORROW_3_r01 1227 -#define _LOAD_FAST_BORROW_3_r12 1228 -#define _LOAD_FAST_BORROW_3_r23 1229 -#define _LOAD_FAST_BORROW_4_r01 1230 -#define _LOAD_FAST_BORROW_4_r12 1231 -#define _LOAD_FAST_BORROW_4_r23 1232 -#define _LOAD_FAST_BORROW_5_r01 1233 -#define _LOAD_FAST_BORROW_5_r12 1234 -#define _LOAD_FAST_BORROW_5_r23 1235 -#define _LOAD_FAST_BORROW_6_r01 1236 -#define _LOAD_FAST_BORROW_6_r12 1237 -#define _LOAD_FAST_BORROW_6_r23 1238 -#define _LOAD_FAST_BORROW_7_r01 1239 -#define _LOAD_FAST_BORROW_7_r12 1240 -#define _LOAD_FAST_BORROW_7_r23 1241 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1242 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1243 -#define _LOAD_FAST_CHECK_r01 1244 -#define _LOAD_FAST_CHECK_r12 1245 -#define _LOAD_FAST_CHECK_r23 1246 -#define _LOAD_FAST_LOAD_FAST_r02 1247 -#define _LOAD_FAST_LOAD_FAST_r13 1248 -#define _LOAD_FROM_DICT_OR_DEREF_r11 1249 -#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1250 -#define _LOAD_GLOBAL_r00 1251 -#define _LOAD_GLOBAL_BUILTINS_r01 1252 -#define _LOAD_GLOBAL_MODULE_r01 1253 -#define _LOAD_LOCALS_r01 1254 -#define _LOAD_LOCALS_r12 1255 -#define _LOAD_LOCALS_r23 1256 -#define _LOAD_NAME_r01 1257 -#define _LOAD_SMALL_INT_r01 1258 -#define _LOAD_SMALL_INT_r12 1259 -#define _LOAD_SMALL_INT_r23 1260 -#define _LOAD_SMALL_INT_0_r01 1261 -#define _LOAD_SMALL_INT_0_r12 1262 -#define _LOAD_SMALL_INT_0_r23 1263 -#define _LOAD_SMALL_INT_1_r01 1264 -#define _LOAD_SMALL_INT_1_r12 1265 -#define _LOAD_SMALL_INT_1_r23 1266 -#define _LOAD_SMALL_INT_2_r01 1267 -#define _LOAD_SMALL_INT_2_r12 1268 -#define _LOAD_SMALL_INT_2_r23 1269 -#define _LOAD_SMALL_INT_3_r01 1270 -#define _LOAD_SMALL_INT_3_r12 1271 -#define _LOAD_SMALL_INT_3_r23 1272 -#define _LOAD_SPECIAL_r00 1273 -#define _LOAD_SUPER_ATTR_ATTR_r31 1274 -#define _LOAD_SUPER_ATTR_METHOD_r32 1275 -#define _LOCK_OBJECT_r01 1276 -#define _LOCK_OBJECT_r11 1277 -#define _LOCK_OBJECT_r22 1278 -#define _LOCK_OBJECT_r33 1279 -#define _MAKE_CALLARGS_A_TUPLE_r33 1280 -#define _MAKE_CELL_r00 1281 -#define _MAKE_FUNCTION_r11 1282 -#define _MAKE_HEAP_SAFE_r01 1283 -#define _MAKE_HEAP_SAFE_r11 1284 -#define _MAKE_HEAP_SAFE_r22 1285 -#define _MAKE_HEAP_SAFE_r33 1286 -#define _MAKE_WARM_r00 1287 -#define _MAKE_WARM_r11 1288 -#define _MAKE_WARM_r22 1289 -#define _MAKE_WARM_r33 1290 -#define _MAP_ADD_r20 1291 -#define _MATCH_CLASS_r33 1292 -#define _MATCH_KEYS_r23 1293 -#define _MATCH_MAPPING_r02 1294 -#define _MATCH_MAPPING_r12 1295 -#define _MATCH_MAPPING_r23 1296 -#define _MATCH_SEQUENCE_r02 1297 -#define _MATCH_SEQUENCE_r12 1298 -#define _MATCH_SEQUENCE_r23 1299 -#define _MAYBE_EXPAND_METHOD_r00 1300 -#define _MAYBE_EXPAND_METHOD_KW_r11 1301 -#define _MONITOR_CALL_r00 1302 -#define _MONITOR_CALL_KW_r11 1303 -#define _MONITOR_JUMP_BACKWARD_r00 1304 -#define _MONITOR_JUMP_BACKWARD_r11 1305 -#define _MONITOR_JUMP_BACKWARD_r22 1306 -#define _MONITOR_JUMP_BACKWARD_r33 1307 -#define _MONITOR_RESUME_r00 1308 -#define _NOP_r00 1309 -#define _NOP_r11 1310 -#define _NOP_r22 1311 -#define _NOP_r33 1312 -#define _POP_CALL_r20 1313 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1314 -#define _POP_CALL_ONE_r30 1315 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1316 -#define _POP_CALL_TWO_r30 1317 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1318 -#define _POP_EXCEPT_r10 1319 -#define _POP_ITER_r20 1320 -#define _POP_JUMP_IF_FALSE_r00 1321 -#define _POP_JUMP_IF_FALSE_r10 1322 -#define _POP_JUMP_IF_FALSE_r21 1323 -#define _POP_JUMP_IF_FALSE_r32 1324 -#define _POP_JUMP_IF_TRUE_r00 1325 -#define _POP_JUMP_IF_TRUE_r10 1326 -#define _POP_JUMP_IF_TRUE_r21 1327 -#define _POP_JUMP_IF_TRUE_r32 1328 -#define _POP_TOP_r10 1329 -#define _POP_TOP_FLOAT_r00 1330 -#define _POP_TOP_FLOAT_r10 1331 -#define _POP_TOP_FLOAT_r21 1332 -#define _POP_TOP_FLOAT_r32 1333 -#define _POP_TOP_INT_r00 1334 -#define _POP_TOP_INT_r10 1335 -#define _POP_TOP_INT_r21 1336 -#define _POP_TOP_INT_r32 1337 -#define _POP_TOP_LOAD_CONST_INLINE_r11 1338 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1339 -#define _POP_TOP_NOP_r00 1340 -#define _POP_TOP_NOP_r10 1341 -#define _POP_TOP_NOP_r21 1342 -#define _POP_TOP_NOP_r32 1343 -#define _POP_TOP_UNICODE_r00 1344 -#define _POP_TOP_UNICODE_r10 1345 -#define _POP_TOP_UNICODE_r21 1346 -#define _POP_TOP_UNICODE_r32 1347 -#define _POP_TWO_r20 1348 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1349 -#define _PUSH_EXC_INFO_r02 1350 -#define _PUSH_EXC_INFO_r12 1351 -#define _PUSH_EXC_INFO_r23 1352 -#define _PUSH_FRAME_r10 1353 -#define _PUSH_NULL_r01 1354 -#define _PUSH_NULL_r12 1355 -#define _PUSH_NULL_r23 1356 -#define _PUSH_NULL_CONDITIONAL_r00 1357 -#define _PY_FRAME_EX_r31 1358 -#define _PY_FRAME_GENERAL_r01 1359 -#define _PY_FRAME_KW_r11 1360 -#define _REPLACE_WITH_TRUE_r02 1361 -#define _REPLACE_WITH_TRUE_r12 1362 -#define _REPLACE_WITH_TRUE_r23 1363 -#define _RESUME_CHECK_r00 1364 -#define _RESUME_CHECK_r11 1365 -#define _RESUME_CHECK_r22 1366 -#define _RESUME_CHECK_r33 1367 -#define _RETURN_GENERATOR_r01 1368 -#define _RETURN_VALUE_r11 1369 -#define _SAVE_RETURN_OFFSET_r00 1370 -#define _SAVE_RETURN_OFFSET_r11 1371 -#define _SAVE_RETURN_OFFSET_r22 1372 -#define _SAVE_RETURN_OFFSET_r33 1373 -#define _SEND_r22 1374 -#define _SEND_GEN_FRAME_r22 1375 -#define _SETUP_ANNOTATIONS_r00 1376 -#define _SET_ADD_r10 1377 -#define _SET_FUNCTION_ATTRIBUTE_r01 1378 -#define _SET_FUNCTION_ATTRIBUTE_r11 1379 -#define _SET_FUNCTION_ATTRIBUTE_r21 1380 -#define _SET_FUNCTION_ATTRIBUTE_r32 1381 -#define _SET_IP_r00 1382 -#define _SET_IP_r11 1383 -#define _SET_IP_r22 1384 -#define _SET_IP_r33 1385 -#define _SET_UPDATE_r11 1386 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02 1387 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12 1388 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22 1389 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32 1390 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03 1391 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13 1392 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23 1393 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 1394 -#define _SPILL_OR_RELOAD_r01 1395 -#define _SPILL_OR_RELOAD_r02 1396 -#define _SPILL_OR_RELOAD_r03 1397 -#define _SPILL_OR_RELOAD_r10 1398 -#define _SPILL_OR_RELOAD_r12 1399 -#define _SPILL_OR_RELOAD_r13 1400 -#define _SPILL_OR_RELOAD_r20 1401 -#define _SPILL_OR_RELOAD_r21 1402 -#define _SPILL_OR_RELOAD_r23 1403 -#define _SPILL_OR_RELOAD_r30 1404 -#define _SPILL_OR_RELOAD_r31 1405 -#define _SPILL_OR_RELOAD_r32 1406 -#define _START_EXECUTOR_r00 1407 -#define _STORE_ATTR_r20 1408 -#define _STORE_ATTR_INSTANCE_VALUE_r21 1409 -#define _STORE_ATTR_SLOT_r21 1410 -#define _STORE_ATTR_WITH_HINT_r21 1411 -#define _STORE_DEREF_r10 1412 -#define _STORE_FAST_LOAD_FAST_r11 1413 -#define _STORE_FAST_STORE_FAST_r20 1414 -#define _STORE_GLOBAL_r10 1415 -#define _STORE_NAME_r10 1416 -#define _STORE_SLICE_r30 1417 -#define _STORE_SUBSCR_r30 1418 -#define _STORE_SUBSCR_DICT_r31 1419 -#define _STORE_SUBSCR_LIST_INT_r32 1420 -#define _SWAP_r11 1421 -#define _SWAP_2_r02 1422 -#define _SWAP_2_r12 1423 -#define _SWAP_2_r22 1424 -#define _SWAP_2_r33 1425 -#define _SWAP_3_r03 1426 -#define _SWAP_3_r13 1427 -#define _SWAP_3_r23 1428 -#define _SWAP_3_r33 1429 -#define _SWAP_FAST_r01 1430 -#define _SWAP_FAST_r11 1431 -#define _SWAP_FAST_r22 1432 -#define _SWAP_FAST_r33 1433 -#define _SWAP_FAST_0_r01 1434 -#define _SWAP_FAST_0_r11 1435 -#define _SWAP_FAST_0_r22 1436 -#define _SWAP_FAST_0_r33 1437 -#define _SWAP_FAST_1_r01 1438 -#define _SWAP_FAST_1_r11 1439 -#define _SWAP_FAST_1_r22 1440 -#define _SWAP_FAST_1_r33 1441 -#define _SWAP_FAST_2_r01 1442 -#define _SWAP_FAST_2_r11 1443 -#define _SWAP_FAST_2_r22 1444 -#define _SWAP_FAST_2_r33 1445 -#define _SWAP_FAST_3_r01 1446 -#define _SWAP_FAST_3_r11 1447 -#define _SWAP_FAST_3_r22 1448 -#define _SWAP_FAST_3_r33 1449 -#define _SWAP_FAST_4_r01 1450 -#define _SWAP_FAST_4_r11 1451 -#define _SWAP_FAST_4_r22 1452 -#define _SWAP_FAST_4_r33 1453 -#define _SWAP_FAST_5_r01 1454 -#define _SWAP_FAST_5_r11 1455 -#define _SWAP_FAST_5_r22 1456 -#define _SWAP_FAST_5_r33 1457 -#define _SWAP_FAST_6_r01 1458 -#define _SWAP_FAST_6_r11 1459 -#define _SWAP_FAST_6_r22 1460 -#define _SWAP_FAST_6_r33 1461 -#define _SWAP_FAST_7_r01 1462 -#define _SWAP_FAST_7_r11 1463 -#define _SWAP_FAST_7_r22 1464 -#define _SWAP_FAST_7_r33 1465 -#define _TIER2_RESUME_CHECK_r00 1466 -#define _TIER2_RESUME_CHECK_r11 1467 -#define _TIER2_RESUME_CHECK_r22 1468 -#define _TIER2_RESUME_CHECK_r33 1469 -#define _TO_BOOL_r11 1470 -#define _TO_BOOL_BOOL_r01 1471 -#define _TO_BOOL_BOOL_r11 1472 -#define _TO_BOOL_BOOL_r22 1473 -#define _TO_BOOL_BOOL_r33 1474 -#define _TO_BOOL_INT_r02 1475 -#define _TO_BOOL_INT_r12 1476 -#define _TO_BOOL_INT_r23 1477 -#define _TO_BOOL_LIST_r02 1478 -#define _TO_BOOL_LIST_r12 1479 -#define _TO_BOOL_LIST_r23 1480 -#define _TO_BOOL_NONE_r01 1481 -#define _TO_BOOL_NONE_r11 1482 -#define _TO_BOOL_NONE_r22 1483 -#define _TO_BOOL_NONE_r33 1484 -#define _TO_BOOL_STR_r02 1485 -#define _TO_BOOL_STR_r12 1486 -#define _TO_BOOL_STR_r23 1487 -#define _TRACE_RECORD_r00 1488 -#define _UNARY_INVERT_r12 1489 -#define _UNARY_NEGATIVE_r12 1490 -#define _UNARY_NOT_r01 1491 -#define _UNARY_NOT_r11 1492 -#define _UNARY_NOT_r22 1493 -#define _UNARY_NOT_r33 1494 -#define _UNPACK_EX_r10 1495 -#define _UNPACK_SEQUENCE_r10 1496 -#define _UNPACK_SEQUENCE_LIST_r10 1497 -#define _UNPACK_SEQUENCE_TUPLE_r10 1498 -#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1499 -#define _WITH_EXCEPT_START_r33 1500 -#define _YIELD_VALUE_r11 1501 -#define MAX_UOP_REGS_ID 1501 +#define _YIELD_VALUE 638 +#define MAX_UOP_ID 638 +#define _BINARY_OP_r23 639 +#define _BINARY_OP_ADD_FLOAT_r03 640 +#define _BINARY_OP_ADD_FLOAT_r13 641 +#define _BINARY_OP_ADD_FLOAT_r23 642 +#define _BINARY_OP_ADD_FLOAT_INPLACE_r03 643 +#define _BINARY_OP_ADD_FLOAT_INPLACE_r13 644 +#define _BINARY_OP_ADD_FLOAT_INPLACE_r23 645 +#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03 646 +#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13 647 +#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23 648 +#define _BINARY_OP_ADD_INT_r03 649 +#define _BINARY_OP_ADD_INT_r13 650 +#define _BINARY_OP_ADD_INT_r23 651 +#define _BINARY_OP_ADD_INT_INPLACE_r03 652 +#define _BINARY_OP_ADD_INT_INPLACE_r13 653 +#define _BINARY_OP_ADD_INT_INPLACE_r23 654 +#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r03 655 +#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r13 656 +#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r23 657 +#define _BINARY_OP_ADD_UNICODE_r03 658 +#define _BINARY_OP_ADD_UNICODE_r13 659 +#define _BINARY_OP_ADD_UNICODE_r23 660 +#define _BINARY_OP_EXTEND_r23 661 +#define _BINARY_OP_INPLACE_ADD_UNICODE_r21 662 +#define _BINARY_OP_MULTIPLY_FLOAT_r03 663 +#define _BINARY_OP_MULTIPLY_FLOAT_r13 664 +#define _BINARY_OP_MULTIPLY_FLOAT_r23 665 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03 666 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13 667 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23 668 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03 669 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13 670 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23 671 +#define _BINARY_OP_MULTIPLY_INT_r03 672 +#define _BINARY_OP_MULTIPLY_INT_r13 673 +#define _BINARY_OP_MULTIPLY_INT_r23 674 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_r03 675 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_r13 676 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_r23 677 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03 678 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13 679 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23 680 +#define _BINARY_OP_SUBSCR_CHECK_FUNC_r23 681 +#define _BINARY_OP_SUBSCR_DICT_r23 682 +#define _BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23 683 +#define _BINARY_OP_SUBSCR_INIT_CALL_r01 684 +#define _BINARY_OP_SUBSCR_INIT_CALL_r11 685 +#define _BINARY_OP_SUBSCR_INIT_CALL_r21 686 +#define _BINARY_OP_SUBSCR_INIT_CALL_r31 687 +#define _BINARY_OP_SUBSCR_LIST_INT_r23 688 +#define _BINARY_OP_SUBSCR_LIST_SLICE_r23 689 +#define _BINARY_OP_SUBSCR_STR_INT_r23 690 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r03 691 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r13 692 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r23 693 +#define _BINARY_OP_SUBSCR_USTR_INT_r23 694 +#define _BINARY_OP_SUBTRACT_FLOAT_r03 695 +#define _BINARY_OP_SUBTRACT_FLOAT_r13 696 +#define _BINARY_OP_SUBTRACT_FLOAT_r23 697 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03 698 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13 699 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23 700 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03 701 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13 702 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23 703 +#define _BINARY_OP_SUBTRACT_INT_r03 704 +#define _BINARY_OP_SUBTRACT_INT_r13 705 +#define _BINARY_OP_SUBTRACT_INT_r23 706 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_r03 707 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_r13 708 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_r23 709 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03 710 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13 711 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23 712 +#define _BINARY_SLICE_r31 713 +#define _BUILD_INTERPOLATION_r01 714 +#define _BUILD_LIST_r01 715 +#define _BUILD_MAP_r01 716 +#define _BUILD_SET_r01 717 +#define _BUILD_SLICE_r01 718 +#define _BUILD_STRING_r01 719 +#define _BUILD_TEMPLATE_r21 720 +#define _BUILD_TUPLE_r01 721 +#define _CALL_BUILTIN_CLASS_r01 722 +#define _CALL_BUILTIN_FAST_r01 723 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS_r01 724 +#define _CALL_BUILTIN_O_r03 725 +#define _CALL_FUNCTION_EX_NON_PY_GENERAL_r31 726 +#define _CALL_INTRINSIC_1_r12 727 +#define _CALL_INTRINSIC_2_r23 728 +#define _CALL_ISINSTANCE_r31 729 +#define _CALL_KW_NON_PY_r11 730 +#define _CALL_LEN_r33 731 +#define _CALL_LIST_APPEND_r03 732 +#define _CALL_LIST_APPEND_r13 733 +#define _CALL_LIST_APPEND_r23 734 +#define _CALL_LIST_APPEND_r33 735 +#define _CALL_METHOD_DESCRIPTOR_FAST_r01 736 +#define _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r01 737 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01 738 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r01 739 +#define _CALL_METHOD_DESCRIPTOR_NOARGS_r01 740 +#define _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r01 741 +#define _CALL_METHOD_DESCRIPTOR_O_r03 742 +#define _CALL_METHOD_DESCRIPTOR_O_INLINE_r03 743 +#define _CALL_NON_PY_GENERAL_r01 744 +#define _CALL_STR_1_r32 745 +#define _CALL_TUPLE_1_r32 746 +#define _CALL_TYPE_1_r02 747 +#define _CALL_TYPE_1_r12 748 +#define _CALL_TYPE_1_r22 749 +#define _CALL_TYPE_1_r32 750 +#define _CHECK_AND_ALLOCATE_OBJECT_r00 751 +#define _CHECK_ATTR_CLASS_r01 752 +#define _CHECK_ATTR_CLASS_r11 753 +#define _CHECK_ATTR_CLASS_r22 754 +#define _CHECK_ATTR_CLASS_r33 755 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r01 756 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r11 757 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r22 758 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r33 759 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00 760 +#define _CHECK_EG_MATCH_r22 761 +#define _CHECK_EXC_MATCH_r22 762 +#define _CHECK_FUNCTION_EXACT_ARGS_r00 763 +#define _CHECK_FUNCTION_VERSION_r00 764 +#define _CHECK_FUNCTION_VERSION_INLINE_r00 765 +#define _CHECK_FUNCTION_VERSION_INLINE_r11 766 +#define _CHECK_FUNCTION_VERSION_INLINE_r22 767 +#define _CHECK_FUNCTION_VERSION_INLINE_r33 768 +#define _CHECK_FUNCTION_VERSION_KW_r11 769 +#define _CHECK_IS_NOT_PY_CALLABLE_r00 770 +#define _CHECK_IS_NOT_PY_CALLABLE_EX_r03 771 +#define _CHECK_IS_NOT_PY_CALLABLE_EX_r13 772 +#define _CHECK_IS_NOT_PY_CALLABLE_EX_r23 773 +#define _CHECK_IS_NOT_PY_CALLABLE_EX_r33 774 +#define _CHECK_IS_NOT_PY_CALLABLE_KW_r11 775 +#define _CHECK_IS_PY_CALLABLE_EX_r03 776 +#define _CHECK_IS_PY_CALLABLE_EX_r13 777 +#define _CHECK_IS_PY_CALLABLE_EX_r23 778 +#define _CHECK_IS_PY_CALLABLE_EX_r33 779 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r01 780 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r11 781 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r22 782 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r33 783 +#define _CHECK_METHOD_VERSION_r00 784 +#define _CHECK_METHOD_VERSION_KW_r11 785 +#define _CHECK_PEP_523_r00 786 +#define _CHECK_PEP_523_r11 787 +#define _CHECK_PEP_523_r22 788 +#define _CHECK_PEP_523_r33 789 +#define _CHECK_PERIODIC_r00 790 +#define _CHECK_PERIODIC_AT_END_r00 791 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00 792 +#define _CHECK_RECURSION_LIMIT_r00 793 +#define _CHECK_RECURSION_LIMIT_r11 794 +#define _CHECK_RECURSION_LIMIT_r22 795 +#define _CHECK_RECURSION_LIMIT_r33 796 +#define _CHECK_RECURSION_REMAINING_r00 797 +#define _CHECK_RECURSION_REMAINING_r11 798 +#define _CHECK_RECURSION_REMAINING_r22 799 +#define _CHECK_RECURSION_REMAINING_r33 800 +#define _CHECK_STACK_SPACE_r00 801 +#define _CHECK_STACK_SPACE_OPERAND_r00 802 +#define _CHECK_STACK_SPACE_OPERAND_r11 803 +#define _CHECK_STACK_SPACE_OPERAND_r22 804 +#define _CHECK_STACK_SPACE_OPERAND_r33 805 +#define _CHECK_VALIDITY_r00 806 +#define _CHECK_VALIDITY_r11 807 +#define _CHECK_VALIDITY_r22 808 +#define _CHECK_VALIDITY_r33 809 +#define _COLD_DYNAMIC_EXIT_r00 810 +#define _COLD_EXIT_r00 811 +#define _COMPARE_OP_r21 812 +#define _COMPARE_OP_FLOAT_r03 813 +#define _COMPARE_OP_FLOAT_r13 814 +#define _COMPARE_OP_FLOAT_r23 815 +#define _COMPARE_OP_INT_r23 816 +#define _COMPARE_OP_STR_r23 817 +#define _CONTAINS_OP_r23 818 +#define _CONTAINS_OP_DICT_r23 819 +#define _CONTAINS_OP_SET_r23 820 +#define _CONVERT_VALUE_r11 821 +#define _COPY_r01 822 +#define _COPY_1_r02 823 +#define _COPY_1_r12 824 +#define _COPY_1_r23 825 +#define _COPY_2_r03 826 +#define _COPY_2_r13 827 +#define _COPY_2_r23 828 +#define _COPY_3_r03 829 +#define _COPY_3_r13 830 +#define _COPY_3_r23 831 +#define _COPY_3_r33 832 +#define _COPY_FREE_VARS_r00 833 +#define _COPY_FREE_VARS_r11 834 +#define _COPY_FREE_VARS_r22 835 +#define _COPY_FREE_VARS_r33 836 +#define _CREATE_INIT_FRAME_r01 837 +#define _DELETE_ATTR_r10 838 +#define _DELETE_DEREF_r00 839 +#define _DELETE_FAST_r00 840 +#define _DELETE_GLOBAL_r00 841 +#define _DELETE_NAME_r00 842 +#define _DELETE_SUBSCR_r20 843 +#define _DEOPT_r00 844 +#define _DEOPT_r10 845 +#define _DEOPT_r20 846 +#define _DEOPT_r30 847 +#define _DICT_MERGE_r11 848 +#define _DICT_UPDATE_r11 849 +#define _DO_CALL_r01 850 +#define _DO_CALL_FUNCTION_EX_r31 851 +#define _DO_CALL_KW_r11 852 +#define _DYNAMIC_EXIT_r00 853 +#define _DYNAMIC_EXIT_r10 854 +#define _DYNAMIC_EXIT_r20 855 +#define _DYNAMIC_EXIT_r30 856 +#define _END_FOR_r10 857 +#define _END_SEND_r31 858 +#define _ERROR_POP_N_r00 859 +#define _EXIT_INIT_CHECK_r10 860 +#define _EXIT_TRACE_r00 861 +#define _EXIT_TRACE_r10 862 +#define _EXIT_TRACE_r20 863 +#define _EXIT_TRACE_r30 864 +#define _EXPAND_METHOD_r00 865 +#define _EXPAND_METHOD_KW_r11 866 +#define _FATAL_ERROR_r00 867 +#define _FATAL_ERROR_r11 868 +#define _FATAL_ERROR_r22 869 +#define _FATAL_ERROR_r33 870 +#define _FORMAT_SIMPLE_r11 871 +#define _FORMAT_WITH_SPEC_r21 872 +#define _FOR_ITER_r23 873 +#define _FOR_ITER_GEN_FRAME_r03 874 +#define _FOR_ITER_GEN_FRAME_r13 875 +#define _FOR_ITER_GEN_FRAME_r23 876 +#define _FOR_ITER_TIER_TWO_r23 877 +#define _GET_AITER_r11 878 +#define _GET_ANEXT_r12 879 +#define _GET_AWAITABLE_r11 880 +#define _GET_ITER_r12 881 +#define _GET_LEN_r12 882 +#define _GUARD_BINARY_OP_EXTEND_r22 883 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02 884 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12 885 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22 886 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33 887 +#define _GUARD_BIT_IS_SET_POP_r00 888 +#define _GUARD_BIT_IS_SET_POP_r10 889 +#define _GUARD_BIT_IS_SET_POP_r21 890 +#define _GUARD_BIT_IS_SET_POP_r32 891 +#define _GUARD_BIT_IS_SET_POP_4_r00 892 +#define _GUARD_BIT_IS_SET_POP_4_r10 893 +#define _GUARD_BIT_IS_SET_POP_4_r21 894 +#define _GUARD_BIT_IS_SET_POP_4_r32 895 +#define _GUARD_BIT_IS_SET_POP_5_r00 896 +#define _GUARD_BIT_IS_SET_POP_5_r10 897 +#define _GUARD_BIT_IS_SET_POP_5_r21 898 +#define _GUARD_BIT_IS_SET_POP_5_r32 899 +#define _GUARD_BIT_IS_SET_POP_6_r00 900 +#define _GUARD_BIT_IS_SET_POP_6_r10 901 +#define _GUARD_BIT_IS_SET_POP_6_r21 902 +#define _GUARD_BIT_IS_SET_POP_6_r32 903 +#define _GUARD_BIT_IS_SET_POP_7_r00 904 +#define _GUARD_BIT_IS_SET_POP_7_r10 905 +#define _GUARD_BIT_IS_SET_POP_7_r21 906 +#define _GUARD_BIT_IS_SET_POP_7_r32 907 +#define _GUARD_BIT_IS_UNSET_POP_r00 908 +#define _GUARD_BIT_IS_UNSET_POP_r10 909 +#define _GUARD_BIT_IS_UNSET_POP_r21 910 +#define _GUARD_BIT_IS_UNSET_POP_r32 911 +#define _GUARD_BIT_IS_UNSET_POP_4_r00 912 +#define _GUARD_BIT_IS_UNSET_POP_4_r10 913 +#define _GUARD_BIT_IS_UNSET_POP_4_r21 914 +#define _GUARD_BIT_IS_UNSET_POP_4_r32 915 +#define _GUARD_BIT_IS_UNSET_POP_5_r00 916 +#define _GUARD_BIT_IS_UNSET_POP_5_r10 917 +#define _GUARD_BIT_IS_UNSET_POP_5_r21 918 +#define _GUARD_BIT_IS_UNSET_POP_5_r32 919 +#define _GUARD_BIT_IS_UNSET_POP_6_r00 920 +#define _GUARD_BIT_IS_UNSET_POP_6_r10 921 +#define _GUARD_BIT_IS_UNSET_POP_6_r21 922 +#define _GUARD_BIT_IS_UNSET_POP_6_r32 923 +#define _GUARD_BIT_IS_UNSET_POP_7_r00 924 +#define _GUARD_BIT_IS_UNSET_POP_7_r10 925 +#define _GUARD_BIT_IS_UNSET_POP_7_r21 926 +#define _GUARD_BIT_IS_UNSET_POP_7_r32 927 +#define _GUARD_CALLABLE_BUILTIN_FAST_r00 928 +#define _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00 929 +#define _GUARD_CALLABLE_BUILTIN_O_r00 930 +#define _GUARD_CALLABLE_ISINSTANCE_r03 931 +#define _GUARD_CALLABLE_ISINSTANCE_r13 932 +#define _GUARD_CALLABLE_ISINSTANCE_r23 933 +#define _GUARD_CALLABLE_ISINSTANCE_r33 934 +#define _GUARD_CALLABLE_LEN_r03 935 +#define _GUARD_CALLABLE_LEN_r13 936 +#define _GUARD_CALLABLE_LEN_r23 937 +#define _GUARD_CALLABLE_LEN_r33 938 +#define _GUARD_CALLABLE_LIST_APPEND_r03 939 +#define _GUARD_CALLABLE_LIST_APPEND_r13 940 +#define _GUARD_CALLABLE_LIST_APPEND_r23 941 +#define _GUARD_CALLABLE_LIST_APPEND_r33 942 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00 943 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 944 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00 945 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00 946 +#define _GUARD_CALLABLE_STR_1_r03 947 +#define _GUARD_CALLABLE_STR_1_r13 948 +#define _GUARD_CALLABLE_STR_1_r23 949 +#define _GUARD_CALLABLE_STR_1_r33 950 +#define _GUARD_CALLABLE_TUPLE_1_r03 951 +#define _GUARD_CALLABLE_TUPLE_1_r13 952 +#define _GUARD_CALLABLE_TUPLE_1_r23 953 +#define _GUARD_CALLABLE_TUPLE_1_r33 954 +#define _GUARD_CALLABLE_TYPE_1_r03 955 +#define _GUARD_CALLABLE_TYPE_1_r13 956 +#define _GUARD_CALLABLE_TYPE_1_r23 957 +#define _GUARD_CALLABLE_TYPE_1_r33 958 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r00 959 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r11 960 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r22 961 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r33 962 +#define _GUARD_CODE_VERSION_RETURN_VALUE_r00 963 +#define _GUARD_CODE_VERSION_RETURN_VALUE_r11 964 +#define _GUARD_CODE_VERSION_RETURN_VALUE_r22 965 +#define _GUARD_CODE_VERSION_RETURN_VALUE_r33 966 +#define _GUARD_CODE_VERSION_YIELD_VALUE_r00 967 +#define _GUARD_CODE_VERSION_YIELD_VALUE_r11 968 +#define _GUARD_CODE_VERSION_YIELD_VALUE_r22 969 +#define _GUARD_CODE_VERSION_YIELD_VALUE_r33 970 +#define _GUARD_CODE_VERSION__PUSH_FRAME_r00 971 +#define _GUARD_CODE_VERSION__PUSH_FRAME_r11 972 +#define _GUARD_CODE_VERSION__PUSH_FRAME_r22 973 +#define _GUARD_CODE_VERSION__PUSH_FRAME_r33 974 +#define _GUARD_DORV_NO_DICT_r01 975 +#define _GUARD_DORV_NO_DICT_r11 976 +#define _GUARD_DORV_NO_DICT_r22 977 +#define _GUARD_DORV_NO_DICT_r33 978 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01 979 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 980 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 981 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 982 +#define _GUARD_GLOBALS_VERSION_r00 983 +#define _GUARD_GLOBALS_VERSION_r11 984 +#define _GUARD_GLOBALS_VERSION_r22 985 +#define _GUARD_GLOBALS_VERSION_r33 986 +#define _GUARD_IP_RETURN_GENERATOR_r00 987 +#define _GUARD_IP_RETURN_GENERATOR_r11 988 +#define _GUARD_IP_RETURN_GENERATOR_r22 989 +#define _GUARD_IP_RETURN_GENERATOR_r33 990 +#define _GUARD_IP_RETURN_VALUE_r00 991 +#define _GUARD_IP_RETURN_VALUE_r11 992 +#define _GUARD_IP_RETURN_VALUE_r22 993 +#define _GUARD_IP_RETURN_VALUE_r33 994 +#define _GUARD_IP_YIELD_VALUE_r00 995 +#define _GUARD_IP_YIELD_VALUE_r11 996 +#define _GUARD_IP_YIELD_VALUE_r22 997 +#define _GUARD_IP_YIELD_VALUE_r33 998 +#define _GUARD_IP__PUSH_FRAME_r00 999 +#define _GUARD_IP__PUSH_FRAME_r11 1000 +#define _GUARD_IP__PUSH_FRAME_r22 1001 +#define _GUARD_IP__PUSH_FRAME_r33 1002 +#define _GUARD_IS_FALSE_POP_r00 1003 +#define _GUARD_IS_FALSE_POP_r10 1004 +#define _GUARD_IS_FALSE_POP_r21 1005 +#define _GUARD_IS_FALSE_POP_r32 1006 +#define _GUARD_IS_NONE_POP_r00 1007 +#define _GUARD_IS_NONE_POP_r10 1008 +#define _GUARD_IS_NONE_POP_r21 1009 +#define _GUARD_IS_NONE_POP_r32 1010 +#define _GUARD_IS_NOT_NONE_POP_r10 1011 +#define _GUARD_IS_TRUE_POP_r00 1012 +#define _GUARD_IS_TRUE_POP_r10 1013 +#define _GUARD_IS_TRUE_POP_r21 1014 +#define _GUARD_IS_TRUE_POP_r32 1015 +#define _GUARD_KEYS_VERSION_r01 1016 +#define _GUARD_KEYS_VERSION_r11 1017 +#define _GUARD_KEYS_VERSION_r22 1018 +#define _GUARD_KEYS_VERSION_r33 1019 +#define _GUARD_NOS_ANY_DICT_r02 1020 +#define _GUARD_NOS_ANY_DICT_r12 1021 +#define _GUARD_NOS_ANY_DICT_r22 1022 +#define _GUARD_NOS_ANY_DICT_r33 1023 +#define _GUARD_NOS_COMPACT_ASCII_r02 1024 +#define _GUARD_NOS_COMPACT_ASCII_r12 1025 +#define _GUARD_NOS_COMPACT_ASCII_r22 1026 +#define _GUARD_NOS_COMPACT_ASCII_r33 1027 +#define _GUARD_NOS_DICT_r02 1028 +#define _GUARD_NOS_DICT_r12 1029 +#define _GUARD_NOS_DICT_r22 1030 +#define _GUARD_NOS_DICT_r33 1031 +#define _GUARD_NOS_FLOAT_r02 1032 +#define _GUARD_NOS_FLOAT_r12 1033 +#define _GUARD_NOS_FLOAT_r22 1034 +#define _GUARD_NOS_FLOAT_r33 1035 +#define _GUARD_NOS_INT_r02 1036 +#define _GUARD_NOS_INT_r12 1037 +#define _GUARD_NOS_INT_r22 1038 +#define _GUARD_NOS_INT_r33 1039 +#define _GUARD_NOS_LIST_r02 1040 +#define _GUARD_NOS_LIST_r12 1041 +#define _GUARD_NOS_LIST_r22 1042 +#define _GUARD_NOS_LIST_r33 1043 +#define _GUARD_NOS_NOT_NULL_r02 1044 +#define _GUARD_NOS_NOT_NULL_r12 1045 +#define _GUARD_NOS_NOT_NULL_r22 1046 +#define _GUARD_NOS_NOT_NULL_r33 1047 +#define _GUARD_NOS_NULL_r02 1048 +#define _GUARD_NOS_NULL_r12 1049 +#define _GUARD_NOS_NULL_r22 1050 +#define _GUARD_NOS_NULL_r33 1051 +#define _GUARD_NOS_OVERFLOWED_r02 1052 +#define _GUARD_NOS_OVERFLOWED_r12 1053 +#define _GUARD_NOS_OVERFLOWED_r22 1054 +#define _GUARD_NOS_OVERFLOWED_r33 1055 +#define _GUARD_NOS_TUPLE_r02 1056 +#define _GUARD_NOS_TUPLE_r12 1057 +#define _GUARD_NOS_TUPLE_r22 1058 +#define _GUARD_NOS_TUPLE_r33 1059 +#define _GUARD_NOS_UNICODE_r02 1060 +#define _GUARD_NOS_UNICODE_r12 1061 +#define _GUARD_NOS_UNICODE_r22 1062 +#define _GUARD_NOS_UNICODE_r33 1063 +#define _GUARD_NOT_EXHAUSTED_LIST_r02 1064 +#define _GUARD_NOT_EXHAUSTED_LIST_r12 1065 +#define _GUARD_NOT_EXHAUSTED_LIST_r22 1066 +#define _GUARD_NOT_EXHAUSTED_LIST_r33 1067 +#define _GUARD_NOT_EXHAUSTED_RANGE_r02 1068 +#define _GUARD_NOT_EXHAUSTED_RANGE_r12 1069 +#define _GUARD_NOT_EXHAUSTED_RANGE_r22 1070 +#define _GUARD_NOT_EXHAUSTED_RANGE_r33 1071 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r02 1072 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r12 1073 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r22 1074 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r33 1075 +#define _GUARD_THIRD_NULL_r03 1076 +#define _GUARD_THIRD_NULL_r13 1077 +#define _GUARD_THIRD_NULL_r23 1078 +#define _GUARD_THIRD_NULL_r33 1079 +#define _GUARD_TOS_ANY_DICT_r01 1080 +#define _GUARD_TOS_ANY_DICT_r11 1081 +#define _GUARD_TOS_ANY_DICT_r22 1082 +#define _GUARD_TOS_ANY_DICT_r33 1083 +#define _GUARD_TOS_ANY_SET_r01 1084 +#define _GUARD_TOS_ANY_SET_r11 1085 +#define _GUARD_TOS_ANY_SET_r22 1086 +#define _GUARD_TOS_ANY_SET_r33 1087 +#define _GUARD_TOS_DICT_r01 1088 +#define _GUARD_TOS_DICT_r11 1089 +#define _GUARD_TOS_DICT_r22 1090 +#define _GUARD_TOS_DICT_r33 1091 +#define _GUARD_TOS_FLOAT_r01 1092 +#define _GUARD_TOS_FLOAT_r11 1093 +#define _GUARD_TOS_FLOAT_r22 1094 +#define _GUARD_TOS_FLOAT_r33 1095 +#define _GUARD_TOS_FROZENDICT_r01 1096 +#define _GUARD_TOS_FROZENDICT_r11 1097 +#define _GUARD_TOS_FROZENDICT_r22 1098 +#define _GUARD_TOS_FROZENDICT_r33 1099 +#define _GUARD_TOS_FROZENSET_r01 1100 +#define _GUARD_TOS_FROZENSET_r11 1101 +#define _GUARD_TOS_FROZENSET_r22 1102 +#define _GUARD_TOS_FROZENSET_r33 1103 +#define _GUARD_TOS_INT_r01 1104 +#define _GUARD_TOS_INT_r11 1105 +#define _GUARD_TOS_INT_r22 1106 +#define _GUARD_TOS_INT_r33 1107 +#define _GUARD_TOS_LIST_r01 1108 +#define _GUARD_TOS_LIST_r11 1109 +#define _GUARD_TOS_LIST_r22 1110 +#define _GUARD_TOS_LIST_r33 1111 +#define _GUARD_TOS_OVERFLOWED_r01 1112 +#define _GUARD_TOS_OVERFLOWED_r11 1113 +#define _GUARD_TOS_OVERFLOWED_r22 1114 +#define _GUARD_TOS_OVERFLOWED_r33 1115 +#define _GUARD_TOS_SET_r01 1116 +#define _GUARD_TOS_SET_r11 1117 +#define _GUARD_TOS_SET_r22 1118 +#define _GUARD_TOS_SET_r33 1119 +#define _GUARD_TOS_SLICE_r01 1120 +#define _GUARD_TOS_SLICE_r11 1121 +#define _GUARD_TOS_SLICE_r22 1122 +#define _GUARD_TOS_SLICE_r33 1123 +#define _GUARD_TOS_TUPLE_r01 1124 +#define _GUARD_TOS_TUPLE_r11 1125 +#define _GUARD_TOS_TUPLE_r22 1126 +#define _GUARD_TOS_TUPLE_r33 1127 +#define _GUARD_TOS_UNICODE_r01 1128 +#define _GUARD_TOS_UNICODE_r11 1129 +#define _GUARD_TOS_UNICODE_r22 1130 +#define _GUARD_TOS_UNICODE_r33 1131 +#define _GUARD_TYPE_VERSION_r01 1132 +#define _GUARD_TYPE_VERSION_r11 1133 +#define _GUARD_TYPE_VERSION_r22 1134 +#define _GUARD_TYPE_VERSION_r33 1135 +#define _GUARD_TYPE_VERSION_LOCKED_r01 1136 +#define _GUARD_TYPE_VERSION_LOCKED_r11 1137 +#define _GUARD_TYPE_VERSION_LOCKED_r22 1138 +#define _GUARD_TYPE_VERSION_LOCKED_r33 1139 +#define _HANDLE_PENDING_AND_DEOPT_r00 1140 +#define _HANDLE_PENDING_AND_DEOPT_r10 1141 +#define _HANDLE_PENDING_AND_DEOPT_r20 1142 +#define _HANDLE_PENDING_AND_DEOPT_r30 1143 +#define _IMPORT_FROM_r12 1144 +#define _IMPORT_NAME_r21 1145 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 1146 +#define _INIT_CALL_PY_EXACT_ARGS_r01 1147 +#define _INIT_CALL_PY_EXACT_ARGS_0_r01 1148 +#define _INIT_CALL_PY_EXACT_ARGS_1_r01 1149 +#define _INIT_CALL_PY_EXACT_ARGS_2_r01 1150 +#define _INIT_CALL_PY_EXACT_ARGS_3_r01 1151 +#define _INIT_CALL_PY_EXACT_ARGS_4_r01 1152 +#define _INSERT_1_LOAD_CONST_INLINE_r02 1153 +#define _INSERT_1_LOAD_CONST_INLINE_r12 1154 +#define _INSERT_1_LOAD_CONST_INLINE_r23 1155 +#define _INSERT_1_LOAD_CONST_INLINE_BORROW_r02 1156 +#define _INSERT_1_LOAD_CONST_INLINE_BORROW_r12 1157 +#define _INSERT_1_LOAD_CONST_INLINE_BORROW_r23 1158 +#define _INSERT_2_LOAD_CONST_INLINE_BORROW_r03 1159 +#define _INSERT_2_LOAD_CONST_INLINE_BORROW_r13 1160 +#define _INSERT_2_LOAD_CONST_INLINE_BORROW_r23 1161 +#define _INSERT_NULL_r10 1162 +#define _INSTRUMENTED_FOR_ITER_r23 1163 +#define _INSTRUMENTED_INSTRUCTION_r00 1164 +#define _INSTRUMENTED_JUMP_FORWARD_r00 1165 +#define _INSTRUMENTED_JUMP_FORWARD_r11 1166 +#define _INSTRUMENTED_JUMP_FORWARD_r22 1167 +#define _INSTRUMENTED_JUMP_FORWARD_r33 1168 +#define _INSTRUMENTED_LINE_r00 1169 +#define _INSTRUMENTED_NOT_TAKEN_r00 1170 +#define _INSTRUMENTED_NOT_TAKEN_r11 1171 +#define _INSTRUMENTED_NOT_TAKEN_r22 1172 +#define _INSTRUMENTED_NOT_TAKEN_r33 1173 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 1174 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 1175 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 1176 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 1177 +#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 1178 +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 1179 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 1180 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 1181 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 1182 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 1183 +#define _IS_NONE_r11 1184 +#define _IS_OP_r03 1185 +#define _IS_OP_r13 1186 +#define _IS_OP_r23 1187 +#define _ITER_CHECK_LIST_r02 1188 +#define _ITER_CHECK_LIST_r12 1189 +#define _ITER_CHECK_LIST_r22 1190 +#define _ITER_CHECK_LIST_r33 1191 +#define _ITER_CHECK_RANGE_r02 1192 +#define _ITER_CHECK_RANGE_r12 1193 +#define _ITER_CHECK_RANGE_r22 1194 +#define _ITER_CHECK_RANGE_r33 1195 +#define _ITER_CHECK_TUPLE_r02 1196 +#define _ITER_CHECK_TUPLE_r12 1197 +#define _ITER_CHECK_TUPLE_r22 1198 +#define _ITER_CHECK_TUPLE_r33 1199 +#define _ITER_JUMP_LIST_r02 1200 +#define _ITER_JUMP_LIST_r12 1201 +#define _ITER_JUMP_LIST_r22 1202 +#define _ITER_JUMP_LIST_r33 1203 +#define _ITER_JUMP_RANGE_r02 1204 +#define _ITER_JUMP_RANGE_r12 1205 +#define _ITER_JUMP_RANGE_r22 1206 +#define _ITER_JUMP_RANGE_r33 1207 +#define _ITER_JUMP_TUPLE_r02 1208 +#define _ITER_JUMP_TUPLE_r12 1209 +#define _ITER_JUMP_TUPLE_r22 1210 +#define _ITER_JUMP_TUPLE_r33 1211 +#define _ITER_NEXT_LIST_r23 1212 +#define _ITER_NEXT_LIST_TIER_TWO_r23 1213 +#define _ITER_NEXT_RANGE_r03 1214 +#define _ITER_NEXT_RANGE_r13 1215 +#define _ITER_NEXT_RANGE_r23 1216 +#define _ITER_NEXT_TUPLE_r03 1217 +#define _ITER_NEXT_TUPLE_r13 1218 +#define _ITER_NEXT_TUPLE_r23 1219 +#define _JUMP_BACKWARD_NO_INTERRUPT_r00 1220 +#define _JUMP_BACKWARD_NO_INTERRUPT_r11 1221 +#define _JUMP_BACKWARD_NO_INTERRUPT_r22 1222 +#define _JUMP_BACKWARD_NO_INTERRUPT_r33 1223 +#define _JUMP_TO_TOP_r00 1224 +#define _LIST_APPEND_r10 1225 +#define _LIST_EXTEND_r11 1226 +#define _LOAD_ATTR_r10 1227 +#define _LOAD_ATTR_CLASS_r11 1228 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_r11 1229 +#define _LOAD_ATTR_INSTANCE_VALUE_r02 1230 +#define _LOAD_ATTR_INSTANCE_VALUE_r12 1231 +#define _LOAD_ATTR_INSTANCE_VALUE_r23 1232 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 1233 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 1234 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 1235 +#define _LOAD_ATTR_METHOD_NO_DICT_r02 1236 +#define _LOAD_ATTR_METHOD_NO_DICT_r12 1237 +#define _LOAD_ATTR_METHOD_NO_DICT_r23 1238 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 1239 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 1240 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1241 +#define _LOAD_ATTR_MODULE_r12 1242 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1243 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1244 +#define _LOAD_ATTR_PROPERTY_FRAME_r11 1245 +#define _LOAD_ATTR_SLOT_r02 1246 +#define _LOAD_ATTR_SLOT_r12 1247 +#define _LOAD_ATTR_SLOT_r23 1248 +#define _LOAD_ATTR_WITH_HINT_r12 1249 +#define _LOAD_BUILD_CLASS_r01 1250 +#define _LOAD_BYTECODE_r00 1251 +#define _LOAD_COMMON_CONSTANT_r01 1252 +#define _LOAD_COMMON_CONSTANT_r12 1253 +#define _LOAD_COMMON_CONSTANT_r23 1254 +#define _LOAD_CONST_r01 1255 +#define _LOAD_CONST_r12 1256 +#define _LOAD_CONST_r23 1257 +#define _LOAD_CONST_INLINE_r01 1258 +#define _LOAD_CONST_INLINE_r12 1259 +#define _LOAD_CONST_INLINE_r23 1260 +#define _LOAD_CONST_INLINE_BORROW_r01 1261 +#define _LOAD_CONST_INLINE_BORROW_r12 1262 +#define _LOAD_CONST_INLINE_BORROW_r23 1263 +#define _LOAD_CONST_UNDER_INLINE_r02 1264 +#define _LOAD_CONST_UNDER_INLINE_r12 1265 +#define _LOAD_CONST_UNDER_INLINE_r23 1266 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r02 1267 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r12 1268 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r23 1269 +#define _LOAD_DEREF_r01 1270 +#define _LOAD_FAST_r01 1271 +#define _LOAD_FAST_r12 1272 +#define _LOAD_FAST_r23 1273 +#define _LOAD_FAST_0_r01 1274 +#define _LOAD_FAST_0_r12 1275 +#define _LOAD_FAST_0_r23 1276 +#define _LOAD_FAST_1_r01 1277 +#define _LOAD_FAST_1_r12 1278 +#define _LOAD_FAST_1_r23 1279 +#define _LOAD_FAST_2_r01 1280 +#define _LOAD_FAST_2_r12 1281 +#define _LOAD_FAST_2_r23 1282 +#define _LOAD_FAST_3_r01 1283 +#define _LOAD_FAST_3_r12 1284 +#define _LOAD_FAST_3_r23 1285 +#define _LOAD_FAST_4_r01 1286 +#define _LOAD_FAST_4_r12 1287 +#define _LOAD_FAST_4_r23 1288 +#define _LOAD_FAST_5_r01 1289 +#define _LOAD_FAST_5_r12 1290 +#define _LOAD_FAST_5_r23 1291 +#define _LOAD_FAST_6_r01 1292 +#define _LOAD_FAST_6_r12 1293 +#define _LOAD_FAST_6_r23 1294 +#define _LOAD_FAST_7_r01 1295 +#define _LOAD_FAST_7_r12 1296 +#define _LOAD_FAST_7_r23 1297 +#define _LOAD_FAST_AND_CLEAR_r01 1298 +#define _LOAD_FAST_AND_CLEAR_r12 1299 +#define _LOAD_FAST_AND_CLEAR_r23 1300 +#define _LOAD_FAST_BORROW_r01 1301 +#define _LOAD_FAST_BORROW_r12 1302 +#define _LOAD_FAST_BORROW_r23 1303 +#define _LOAD_FAST_BORROW_0_r01 1304 +#define _LOAD_FAST_BORROW_0_r12 1305 +#define _LOAD_FAST_BORROW_0_r23 1306 +#define _LOAD_FAST_BORROW_1_r01 1307 +#define _LOAD_FAST_BORROW_1_r12 1308 +#define _LOAD_FAST_BORROW_1_r23 1309 +#define _LOAD_FAST_BORROW_2_r01 1310 +#define _LOAD_FAST_BORROW_2_r12 1311 +#define _LOAD_FAST_BORROW_2_r23 1312 +#define _LOAD_FAST_BORROW_3_r01 1313 +#define _LOAD_FAST_BORROW_3_r12 1314 +#define _LOAD_FAST_BORROW_3_r23 1315 +#define _LOAD_FAST_BORROW_4_r01 1316 +#define _LOAD_FAST_BORROW_4_r12 1317 +#define _LOAD_FAST_BORROW_4_r23 1318 +#define _LOAD_FAST_BORROW_5_r01 1319 +#define _LOAD_FAST_BORROW_5_r12 1320 +#define _LOAD_FAST_BORROW_5_r23 1321 +#define _LOAD_FAST_BORROW_6_r01 1322 +#define _LOAD_FAST_BORROW_6_r12 1323 +#define _LOAD_FAST_BORROW_6_r23 1324 +#define _LOAD_FAST_BORROW_7_r01 1325 +#define _LOAD_FAST_BORROW_7_r12 1326 +#define _LOAD_FAST_BORROW_7_r23 1327 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1328 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1329 +#define _LOAD_FAST_CHECK_r01 1330 +#define _LOAD_FAST_CHECK_r12 1331 +#define _LOAD_FAST_CHECK_r23 1332 +#define _LOAD_FAST_LOAD_FAST_r02 1333 +#define _LOAD_FAST_LOAD_FAST_r13 1334 +#define _LOAD_FROM_DICT_OR_DEREF_r11 1335 +#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1336 +#define _LOAD_GLOBAL_r00 1337 +#define _LOAD_GLOBAL_BUILTINS_r01 1338 +#define _LOAD_GLOBAL_MODULE_r01 1339 +#define _LOAD_LOCALS_r01 1340 +#define _LOAD_LOCALS_r12 1341 +#define _LOAD_LOCALS_r23 1342 +#define _LOAD_NAME_r01 1343 +#define _LOAD_SMALL_INT_r01 1344 +#define _LOAD_SMALL_INT_r12 1345 +#define _LOAD_SMALL_INT_r23 1346 +#define _LOAD_SMALL_INT_0_r01 1347 +#define _LOAD_SMALL_INT_0_r12 1348 +#define _LOAD_SMALL_INT_0_r23 1349 +#define _LOAD_SMALL_INT_1_r01 1350 +#define _LOAD_SMALL_INT_1_r12 1351 +#define _LOAD_SMALL_INT_1_r23 1352 +#define _LOAD_SMALL_INT_2_r01 1353 +#define _LOAD_SMALL_INT_2_r12 1354 +#define _LOAD_SMALL_INT_2_r23 1355 +#define _LOAD_SMALL_INT_3_r01 1356 +#define _LOAD_SMALL_INT_3_r12 1357 +#define _LOAD_SMALL_INT_3_r23 1358 +#define _LOAD_SPECIAL_r00 1359 +#define _LOAD_SUPER_ATTR_ATTR_r31 1360 +#define _LOAD_SUPER_ATTR_METHOD_r32 1361 +#define _LOCK_OBJECT_r01 1362 +#define _LOCK_OBJECT_r11 1363 +#define _LOCK_OBJECT_r22 1364 +#define _LOCK_OBJECT_r33 1365 +#define _MAKE_CALLARGS_A_TUPLE_r33 1366 +#define _MAKE_CELL_r00 1367 +#define _MAKE_FUNCTION_r11 1368 +#define _MAKE_HEAP_SAFE_r01 1369 +#define _MAKE_HEAP_SAFE_r11 1370 +#define _MAKE_HEAP_SAFE_r22 1371 +#define _MAKE_HEAP_SAFE_r33 1372 +#define _MAKE_WARM_r00 1373 +#define _MAKE_WARM_r11 1374 +#define _MAKE_WARM_r22 1375 +#define _MAKE_WARM_r33 1376 +#define _MAP_ADD_r20 1377 +#define _MATCH_CLASS_r33 1378 +#define _MATCH_KEYS_r23 1379 +#define _MATCH_MAPPING_r02 1380 +#define _MATCH_MAPPING_r12 1381 +#define _MATCH_MAPPING_r23 1382 +#define _MATCH_SEQUENCE_r02 1383 +#define _MATCH_SEQUENCE_r12 1384 +#define _MATCH_SEQUENCE_r23 1385 +#define _MAYBE_EXPAND_METHOD_r00 1386 +#define _MAYBE_EXPAND_METHOD_KW_r11 1387 +#define _MONITOR_CALL_r00 1388 +#define _MONITOR_CALL_KW_r11 1389 +#define _MONITOR_JUMP_BACKWARD_r00 1390 +#define _MONITOR_JUMP_BACKWARD_r11 1391 +#define _MONITOR_JUMP_BACKWARD_r22 1392 +#define _MONITOR_JUMP_BACKWARD_r33 1393 +#define _MONITOR_RESUME_r00 1394 +#define _NOP_r00 1395 +#define _NOP_r11 1396 +#define _NOP_r22 1397 +#define _NOP_r33 1398 +#define _POP_CALL_r20 1399 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1400 +#define _POP_CALL_ONE_r30 1401 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1402 +#define _POP_CALL_TWO_r30 1403 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1404 +#define _POP_EXCEPT_r10 1405 +#define _POP_ITER_r20 1406 +#define _POP_JUMP_IF_FALSE_r00 1407 +#define _POP_JUMP_IF_FALSE_r10 1408 +#define _POP_JUMP_IF_FALSE_r21 1409 +#define _POP_JUMP_IF_FALSE_r32 1410 +#define _POP_JUMP_IF_TRUE_r00 1411 +#define _POP_JUMP_IF_TRUE_r10 1412 +#define _POP_JUMP_IF_TRUE_r21 1413 +#define _POP_JUMP_IF_TRUE_r32 1414 +#define _POP_TOP_r10 1415 +#define _POP_TOP_FLOAT_r00 1416 +#define _POP_TOP_FLOAT_r10 1417 +#define _POP_TOP_FLOAT_r21 1418 +#define _POP_TOP_FLOAT_r32 1419 +#define _POP_TOP_INT_r00 1420 +#define _POP_TOP_INT_r10 1421 +#define _POP_TOP_INT_r21 1422 +#define _POP_TOP_INT_r32 1423 +#define _POP_TOP_LOAD_CONST_INLINE_r11 1424 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1425 +#define _POP_TOP_NOP_r00 1426 +#define _POP_TOP_NOP_r10 1427 +#define _POP_TOP_NOP_r21 1428 +#define _POP_TOP_NOP_r32 1429 +#define _POP_TOP_UNICODE_r00 1430 +#define _POP_TOP_UNICODE_r10 1431 +#define _POP_TOP_UNICODE_r21 1432 +#define _POP_TOP_UNICODE_r32 1433 +#define _POP_TWO_r20 1434 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1435 +#define _PUSH_EXC_INFO_r02 1436 +#define _PUSH_EXC_INFO_r12 1437 +#define _PUSH_EXC_INFO_r23 1438 +#define _PUSH_FRAME_r10 1439 +#define _PUSH_NULL_r01 1440 +#define _PUSH_NULL_r12 1441 +#define _PUSH_NULL_r23 1442 +#define _PUSH_NULL_CONDITIONAL_r00 1443 +#define _PY_FRAME_EX_r31 1444 +#define _PY_FRAME_GENERAL_r01 1445 +#define _PY_FRAME_KW_r11 1446 +#define _REPLACE_WITH_TRUE_r02 1447 +#define _REPLACE_WITH_TRUE_r12 1448 +#define _REPLACE_WITH_TRUE_r23 1449 +#define _RESUME_CHECK_r00 1450 +#define _RESUME_CHECK_r11 1451 +#define _RESUME_CHECK_r22 1452 +#define _RESUME_CHECK_r33 1453 +#define _RETURN_GENERATOR_r01 1454 +#define _RETURN_VALUE_r11 1455 +#define _SAVE_RETURN_OFFSET_r00 1456 +#define _SAVE_RETURN_OFFSET_r11 1457 +#define _SAVE_RETURN_OFFSET_r22 1458 +#define _SAVE_RETURN_OFFSET_r33 1459 +#define _SEND_r33 1460 +#define _SEND_GEN_FRAME_r33 1461 +#define _SETUP_ANNOTATIONS_r00 1462 +#define _SET_ADD_r10 1463 +#define _SET_FUNCTION_ATTRIBUTE_r01 1464 +#define _SET_FUNCTION_ATTRIBUTE_r11 1465 +#define _SET_FUNCTION_ATTRIBUTE_r21 1466 +#define _SET_FUNCTION_ATTRIBUTE_r32 1467 +#define _SET_IP_r00 1468 +#define _SET_IP_r11 1469 +#define _SET_IP_r22 1470 +#define _SET_IP_r33 1471 +#define _SET_UPDATE_r11 1472 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02 1473 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12 1474 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22 1475 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32 1476 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03 1477 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13 1478 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23 1479 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 1480 +#define _SPILL_OR_RELOAD_r01 1481 +#define _SPILL_OR_RELOAD_r02 1482 +#define _SPILL_OR_RELOAD_r03 1483 +#define _SPILL_OR_RELOAD_r10 1484 +#define _SPILL_OR_RELOAD_r12 1485 +#define _SPILL_OR_RELOAD_r13 1486 +#define _SPILL_OR_RELOAD_r20 1487 +#define _SPILL_OR_RELOAD_r21 1488 +#define _SPILL_OR_RELOAD_r23 1489 +#define _SPILL_OR_RELOAD_r30 1490 +#define _SPILL_OR_RELOAD_r31 1491 +#define _SPILL_OR_RELOAD_r32 1492 +#define _START_EXECUTOR_r00 1493 +#define _STORE_ATTR_r20 1494 +#define _STORE_ATTR_INSTANCE_VALUE_r21 1495 +#define _STORE_ATTR_SLOT_r21 1496 +#define _STORE_ATTR_WITH_HINT_r21 1497 +#define _STORE_DEREF_r10 1498 +#define _STORE_FAST_LOAD_FAST_r11 1499 +#define _STORE_FAST_STORE_FAST_r20 1500 +#define _STORE_GLOBAL_r10 1501 +#define _STORE_NAME_r10 1502 +#define _STORE_SLICE_r30 1503 +#define _STORE_SUBSCR_r30 1504 +#define _STORE_SUBSCR_DICT_r31 1505 +#define _STORE_SUBSCR_DICT_KNOWN_HASH_r31 1506 +#define _STORE_SUBSCR_LIST_INT_r32 1507 +#define _SWAP_r11 1508 +#define _SWAP_2_r02 1509 +#define _SWAP_2_r12 1510 +#define _SWAP_2_r22 1511 +#define _SWAP_2_r33 1512 +#define _SWAP_3_r03 1513 +#define _SWAP_3_r13 1514 +#define _SWAP_3_r23 1515 +#define _SWAP_3_r33 1516 +#define _SWAP_FAST_r01 1517 +#define _SWAP_FAST_r11 1518 +#define _SWAP_FAST_r22 1519 +#define _SWAP_FAST_r33 1520 +#define _SWAP_FAST_0_r01 1521 +#define _SWAP_FAST_0_r11 1522 +#define _SWAP_FAST_0_r22 1523 +#define _SWAP_FAST_0_r33 1524 +#define _SWAP_FAST_1_r01 1525 +#define _SWAP_FAST_1_r11 1526 +#define _SWAP_FAST_1_r22 1527 +#define _SWAP_FAST_1_r33 1528 +#define _SWAP_FAST_2_r01 1529 +#define _SWAP_FAST_2_r11 1530 +#define _SWAP_FAST_2_r22 1531 +#define _SWAP_FAST_2_r33 1532 +#define _SWAP_FAST_3_r01 1533 +#define _SWAP_FAST_3_r11 1534 +#define _SWAP_FAST_3_r22 1535 +#define _SWAP_FAST_3_r33 1536 +#define _SWAP_FAST_4_r01 1537 +#define _SWAP_FAST_4_r11 1538 +#define _SWAP_FAST_4_r22 1539 +#define _SWAP_FAST_4_r33 1540 +#define _SWAP_FAST_5_r01 1541 +#define _SWAP_FAST_5_r11 1542 +#define _SWAP_FAST_5_r22 1543 +#define _SWAP_FAST_5_r33 1544 +#define _SWAP_FAST_6_r01 1545 +#define _SWAP_FAST_6_r11 1546 +#define _SWAP_FAST_6_r22 1547 +#define _SWAP_FAST_6_r33 1548 +#define _SWAP_FAST_7_r01 1549 +#define _SWAP_FAST_7_r11 1550 +#define _SWAP_FAST_7_r22 1551 +#define _SWAP_FAST_7_r33 1552 +#define _TIER2_RESUME_CHECK_r00 1553 +#define _TIER2_RESUME_CHECK_r11 1554 +#define _TIER2_RESUME_CHECK_r22 1555 +#define _TIER2_RESUME_CHECK_r33 1556 +#define _TO_BOOL_r11 1557 +#define _TO_BOOL_BOOL_r01 1558 +#define _TO_BOOL_BOOL_r11 1559 +#define _TO_BOOL_BOOL_r22 1560 +#define _TO_BOOL_BOOL_r33 1561 +#define _TO_BOOL_INT_r02 1562 +#define _TO_BOOL_INT_r12 1563 +#define _TO_BOOL_INT_r23 1564 +#define _TO_BOOL_LIST_r02 1565 +#define _TO_BOOL_LIST_r12 1566 +#define _TO_BOOL_LIST_r23 1567 +#define _TO_BOOL_NONE_r01 1568 +#define _TO_BOOL_NONE_r11 1569 +#define _TO_BOOL_NONE_r22 1570 +#define _TO_BOOL_NONE_r33 1571 +#define _TO_BOOL_STR_r02 1572 +#define _TO_BOOL_STR_r12 1573 +#define _TO_BOOL_STR_r23 1574 +#define _TRACE_RECORD_r00 1575 +#define _UNARY_INVERT_r12 1576 +#define _UNARY_NEGATIVE_r12 1577 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r02 1578 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r12 1579 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r23 1580 +#define _UNARY_NOT_r01 1581 +#define _UNARY_NOT_r11 1582 +#define _UNARY_NOT_r22 1583 +#define _UNARY_NOT_r33 1584 +#define _UNPACK_EX_r10 1585 +#define _UNPACK_SEQUENCE_r10 1586 +#define _UNPACK_SEQUENCE_LIST_r10 1587 +#define _UNPACK_SEQUENCE_TUPLE_r10 1588 +#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1589 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03 1590 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13 1591 +#define _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10 1592 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02 1593 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12 1594 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23 1595 +#define _WITH_EXCEPT_START_r33 1596 +#define _YIELD_VALUE_r11 1597 +#define MAX_UOP_REGS_ID 1597 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 744f6438659db8..016bed17bf15aa 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -85,6 +85,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_POP_ITER] = HAS_ESCAPES_FLAG, [_END_SEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_UNARY_NEGATIVE_FLOAT_INPLACE] = 0, [_UNARY_NOT] = 0, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_TO_BOOL_BOOL] = HAS_EXIT_FLAG, @@ -107,11 +108,23 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_MULTIPLY_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_INT_INPLACE] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBTRACT_INT_INPLACE] = HAS_EXIT_FLAG, + [_BINARY_OP_MULTIPLY_INT_INPLACE] = HAS_EXIT_FLAG, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT] = HAS_EXIT_FLAG, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT] = HAS_EXIT_FLAG, [_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG, [_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG, [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_FLOAT_INPLACE] = 0, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE] = 0, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE] = 0, + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT] = 0, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT] = 0, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT] = 0, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_BINARY_OP_EXTEND] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, @@ -131,6 +144,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_TOS_ANY_DICT] = HAS_EXIT_FLAG, [_GUARD_TOS_DICT] = HAS_EXIT_FLAG, [_GUARD_TOS_FROZENDICT] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBSCR_DICT_KNOWN_HASH] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_CHECK_FUNC] = HAS_EXIT_FLAG, [_BINARY_OP_SUBSCR_INIT_CALL] = 0, @@ -139,9 +153,10 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_SUBSCR_DICT_KNOWN_HASH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAKE_HEAP_SAFE] = 0, [_RETURN_VALUE] = HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -156,7 +171,10 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE] = 0, + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE] = 0, [_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_UNPACK_SEQUENCE_UNIQUE_TUPLE] = HAS_ARG_FLAG, [_UNPACK_SEQUENCE_LIST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_EX] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -182,13 +200,13 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BUILD_TEMPLATE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_TUPLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, [_BUILD_LIST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_LIST_EXTEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LIST_EXTEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_SET_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_BUILD_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SETUP_ANNOTATIONS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DICT_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DICT_MERGE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DICT_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_DICT_MERGE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAP_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, @@ -229,8 +247,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MATCH_MAPPING] = 0, [_MATCH_SEQUENCE] = 0, [_MATCH_KEYS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GET_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GET_ITER] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, @@ -288,19 +305,31 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CREATE_INIT_FRAME] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_BUILTIN_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_CALLABLE_LEN] = HAS_EXIT_FLAG, [_CALL_LEN] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_CALLABLE_ISINSTANCE] = HAS_EXIT_FLAG, [_CALL_ISINSTANCE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_CALLABLE_LIST_APPEND] = HAS_EXIT_FLAG, [_CALL_LIST_APPEND] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, - [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_RECURSION_LIMIT] = HAS_EXIT_FLAG, + [_CALL_METHOD_DESCRIPTOR_O_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAYBE_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_CHECK_FUNCTION_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, @@ -389,6 +418,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_RECORD_TOS_TYPE] = HAS_RECORDS_VALUE_FLAG, [_RECORD_NOS] = HAS_RECORDS_VALUE_FLAG, [_RECORD_NOS_GEN_FUNC] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_3OS_GEN_FUNC] = HAS_RECORDS_VALUE_FLAG, [_RECORD_4OS] = HAS_RECORDS_VALUE_FLAG, [_RECORD_CALLABLE] = HAS_ARG_FLAG | HAS_RECORDS_VALUE_FLAG, [_RECORD_BOUND_METHOD] = HAS_ARG_FLAG | HAS_RECORDS_VALUE_FLAG, @@ -841,12 +871,12 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_END_SEND] = { - .best = { 2, 2, 2, 2 }, + .best = { 3, 3, 3, 3 }, .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 2, _END_SEND_r21 }, { -1, -1, -1 }, + { 1, 3, _END_SEND_r31 }, }, }, [_UNARY_NEGATIVE] = { @@ -858,6 +888,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_UNARY_NEGATIVE_FLOAT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _UNARY_NEGATIVE_FLOAT_INPLACE_r02 }, + { 2, 1, _UNARY_NEGATIVE_FLOAT_INPLACE_r12 }, + { 3, 2, _UNARY_NEGATIVE_FLOAT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, [_UNARY_NOT] = { .best = { 0, 1, 2, 3 }, .entries = { @@ -1056,6 +1095,60 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_BINARY_OP_ADD_INT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_INT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_ADD_INT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_ADD_INT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_INT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_INT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_INT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_INT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_MULTIPLY_INT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_INT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_INT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_INT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_INT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_ADD_INT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_ADD_INT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, [_GUARD_NOS_FLOAT] = { .best = { 0, 1, 2, 3 }, .entries = { @@ -1101,6 +1194,60 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_BINARY_OP_ADD_FLOAT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_FLOAT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_ADD_FLOAT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_ADD_FLOAT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, [_BINARY_OP_ADD_UNICODE] = { .best = { 0, 1, 2, 2 }, .entries = { @@ -1272,6 +1419,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 3, 3, _GUARD_TOS_FROZENDICT_r33 }, }, }, + [_BINARY_OP_SUBSCR_DICT_KNOWN_HASH] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23 }, + { -1, -1, -1 }, + }, + }, [_BINARY_OP_SUBSCR_DICT] = { .best = { 2, 2, 2, 2 }, .entries = { @@ -1344,6 +1500,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 1, 3, _STORE_SUBSCR_DICT_r31 }, }, }, + [_STORE_SUBSCR_DICT_KNOWN_HASH] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _STORE_SUBSCR_DICT_KNOWN_HASH_r31 }, + }, + }, [_DELETE_SUBSCR] = { .best = { 2, 2, 2, 2 }, .entries = { @@ -1367,7 +1532,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 2, _CALL_INTRINSIC_2_r21 }, + { 3, 2, _CALL_INTRINSIC_2_r23 }, { -1, -1, -1 }, }, }, @@ -1417,12 +1582,12 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_SEND_GEN_FRAME] = { - .best = { 2, 2, 2, 2 }, + .best = { 3, 3, 3, 3 }, .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 2, 2, _SEND_GEN_FRAME_r22 }, { -1, -1, -1 }, + { 3, 3, _SEND_GEN_FRAME_r33 }, }, }, [_YIELD_VALUE] = { @@ -1497,6 +1662,24 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02 }, + { 2, 1, _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12 }, + { 3, 2, _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE] = { + .best = { 0, 1, 1, 1 }, + .entries = { + { 3, 0, _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03 }, + { 3, 1, _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_UNPACK_SEQUENCE_TUPLE] = { .best = { 1, 1, 1, 1 }, .entries = { @@ -1506,6 +1689,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_UNPACK_SEQUENCE_UNIQUE_TUPLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_UNPACK_SEQUENCE_LIST] = { .best = { 1, 1, 1, 1 }, .entries = { @@ -1735,7 +1927,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 0, 1, _LIST_EXTEND_r10 }, + { 1, 1, _LIST_EXTEND_r11 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1780,7 +1972,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 0, 1, _DICT_UPDATE_r10 }, + { 1, 1, _DICT_UPDATE_r11 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -1789,7 +1981,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 0, 1, _DICT_MERGE_r10 }, + { 1, 1, _DICT_MERGE_r11 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -2163,15 +2355,6 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, - [_GET_YIELD_FROM_ITER] = { - .best = { 1, 1, 1, 1 }, - .entries = { - { -1, -1, -1 }, - { 1, 1, _GET_YIELD_FROM_ITER_r11 }, - { -1, -1, -1 }, - { -1, -1, -1 }, - }, - }, [_FOR_ITER_TIER_TWO] = { .best = { 2, 2, 2, 2 }, .entries = { @@ -2685,6 +2868,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_GUARD_CALLABLE_BUILTIN_O] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_BUILTIN_O_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_CALL_BUILTIN_O] = { .best = { 0, 0, 0, 0 }, .entries = { @@ -2694,6 +2886,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_GUARD_CALLABLE_BUILTIN_FAST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_BUILTIN_FAST_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_CALL_BUILTIN_FAST] = { .best = { 0, 0, 0, 0 }, .entries = { @@ -2703,6 +2904,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .best = { 0, 0, 0, 0 }, .entries = { @@ -2766,6 +2976,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 3, 3, _CALL_LIST_APPEND_r33 }, }, }, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_O] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_CALL_METHOD_DESCRIPTOR_O] = { .best = { 0, 0, 0, 0 }, .entries = { @@ -2775,6 +2994,33 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_CHECK_RECURSION_LIMIT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_RECURSION_LIMIT_r00 }, + { 1, 1, _CHECK_RECURSION_LIMIT_r11 }, + { 2, 2, _CHECK_RECURSION_LIMIT_r22 }, + { 3, 3, _CHECK_RECURSION_LIMIT_r33 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_O_INLINE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 3, 0, _CALL_METHOD_DESCRIPTOR_O_INLINE_r03 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .best = { 0, 0, 0, 0 }, .entries = { @@ -2784,6 +3030,24 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_CALL_METHOD_DESCRIPTOR_NOARGS] = { .best = { 0, 0, 0, 0 }, .entries = { @@ -2793,6 +3057,24 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_CALL_METHOD_DESCRIPTOR_FAST] = { .best = { 0, 0, 0, 0 }, .entries = { @@ -2802,6 +3084,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_MAYBE_EXPAND_METHOD_KW] = { .best = { 1, 1, 1, 1 }, .entries = { @@ -3699,8 +3990,11 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_PUSH_NULL_r23] = _PUSH_NULL, [_END_FOR_r10] = _END_FOR, [_POP_ITER_r20] = _POP_ITER, - [_END_SEND_r21] = _END_SEND, + [_END_SEND_r31] = _END_SEND, [_UNARY_NEGATIVE_r12] = _UNARY_NEGATIVE, + [_UNARY_NEGATIVE_FLOAT_INPLACE_r02] = _UNARY_NEGATIVE_FLOAT_INPLACE, + [_UNARY_NEGATIVE_FLOAT_INPLACE_r12] = _UNARY_NEGATIVE_FLOAT_INPLACE, + [_UNARY_NEGATIVE_FLOAT_INPLACE_r23] = _UNARY_NEGATIVE_FLOAT_INPLACE, [_UNARY_NOT_r01] = _UNARY_NOT, [_UNARY_NOT_r11] = _UNARY_NOT, [_UNARY_NOT_r22] = _UNARY_NOT, @@ -3776,6 +4070,24 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_BINARY_OP_SUBTRACT_INT_r03] = _BINARY_OP_SUBTRACT_INT, [_BINARY_OP_SUBTRACT_INT_r13] = _BINARY_OP_SUBTRACT_INT, [_BINARY_OP_SUBTRACT_INT_r23] = _BINARY_OP_SUBTRACT_INT, + [_BINARY_OP_ADD_INT_INPLACE_r03] = _BINARY_OP_ADD_INT_INPLACE, + [_BINARY_OP_ADD_INT_INPLACE_r13] = _BINARY_OP_ADD_INT_INPLACE, + [_BINARY_OP_ADD_INT_INPLACE_r23] = _BINARY_OP_ADD_INT_INPLACE, + [_BINARY_OP_SUBTRACT_INT_INPLACE_r03] = _BINARY_OP_SUBTRACT_INT_INPLACE, + [_BINARY_OP_SUBTRACT_INT_INPLACE_r13] = _BINARY_OP_SUBTRACT_INT_INPLACE, + [_BINARY_OP_SUBTRACT_INT_INPLACE_r23] = _BINARY_OP_SUBTRACT_INT_INPLACE, + [_BINARY_OP_MULTIPLY_INT_INPLACE_r03] = _BINARY_OP_MULTIPLY_INT_INPLACE, + [_BINARY_OP_MULTIPLY_INT_INPLACE_r13] = _BINARY_OP_MULTIPLY_INT_INPLACE, + [_BINARY_OP_MULTIPLY_INT_INPLACE_r23] = _BINARY_OP_MULTIPLY_INT_INPLACE, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r03] = _BINARY_OP_ADD_INT_INPLACE_RIGHT, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r13] = _BINARY_OP_ADD_INT_INPLACE_RIGHT, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r23] = _BINARY_OP_ADD_INT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03] = _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13] = _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23] = _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03] = _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13] = _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23] = _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, [_GUARD_NOS_FLOAT_r02] = _GUARD_NOS_FLOAT, [_GUARD_NOS_FLOAT_r12] = _GUARD_NOS_FLOAT, [_GUARD_NOS_FLOAT_r22] = _GUARD_NOS_FLOAT, @@ -3793,6 +4105,24 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_BINARY_OP_SUBTRACT_FLOAT_r03] = _BINARY_OP_SUBTRACT_FLOAT, [_BINARY_OP_SUBTRACT_FLOAT_r13] = _BINARY_OP_SUBTRACT_FLOAT, [_BINARY_OP_SUBTRACT_FLOAT_r23] = _BINARY_OP_SUBTRACT_FLOAT, + [_BINARY_OP_ADD_FLOAT_INPLACE_r03] = _BINARY_OP_ADD_FLOAT_INPLACE, + [_BINARY_OP_ADD_FLOAT_INPLACE_r13] = _BINARY_OP_ADD_FLOAT_INPLACE, + [_BINARY_OP_ADD_FLOAT_INPLACE_r23] = _BINARY_OP_ADD_FLOAT_INPLACE, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE, + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03] = _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13] = _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23] = _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, [_BINARY_OP_ADD_UNICODE_r03] = _BINARY_OP_ADD_UNICODE, [_BINARY_OP_ADD_UNICODE_r13] = _BINARY_OP_ADD_UNICODE, [_BINARY_OP_ADD_UNICODE_r23] = _BINARY_OP_ADD_UNICODE, @@ -3840,6 +4170,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_TOS_FROZENDICT_r11] = _GUARD_TOS_FROZENDICT, [_GUARD_TOS_FROZENDICT_r22] = _GUARD_TOS_FROZENDICT, [_GUARD_TOS_FROZENDICT_r33] = _GUARD_TOS_FROZENDICT, + [_BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23] = _BINARY_OP_SUBSCR_DICT_KNOWN_HASH, [_BINARY_OP_SUBSCR_DICT_r23] = _BINARY_OP_SUBSCR_DICT, [_BINARY_OP_SUBSCR_CHECK_FUNC_r23] = _BINARY_OP_SUBSCR_CHECK_FUNC, [_BINARY_OP_SUBSCR_INIT_CALL_r01] = _BINARY_OP_SUBSCR_INIT_CALL, @@ -3851,9 +4182,10 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_STORE_SUBSCR_r30] = _STORE_SUBSCR, [_STORE_SUBSCR_LIST_INT_r32] = _STORE_SUBSCR_LIST_INT, [_STORE_SUBSCR_DICT_r31] = _STORE_SUBSCR_DICT, + [_STORE_SUBSCR_DICT_KNOWN_HASH_r31] = _STORE_SUBSCR_DICT_KNOWN_HASH, [_DELETE_SUBSCR_r20] = _DELETE_SUBSCR, [_CALL_INTRINSIC_1_r12] = _CALL_INTRINSIC_1, - [_CALL_INTRINSIC_2_r21] = _CALL_INTRINSIC_2, + [_CALL_INTRINSIC_2_r23] = _CALL_INTRINSIC_2, [_MAKE_HEAP_SAFE_r01] = _MAKE_HEAP_SAFE, [_MAKE_HEAP_SAFE_r11] = _MAKE_HEAP_SAFE, [_MAKE_HEAP_SAFE_r22] = _MAKE_HEAP_SAFE, @@ -3862,7 +4194,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GET_AITER_r11] = _GET_AITER, [_GET_ANEXT_r12] = _GET_ANEXT, [_GET_AWAITABLE_r11] = _GET_AWAITABLE, - [_SEND_GEN_FRAME_r22] = _SEND_GEN_FRAME, + [_SEND_GEN_FRAME_r33] = _SEND_GEN_FRAME, [_YIELD_VALUE_r11] = _YIELD_VALUE, [_POP_EXCEPT_r10] = _POP_EXCEPT, [_LOAD_COMMON_CONSTANT_r01] = _LOAD_COMMON_CONSTANT, @@ -3873,7 +4205,13 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_DELETE_NAME_r00] = _DELETE_NAME, [_UNPACK_SEQUENCE_r10] = _UNPACK_SEQUENCE, [_UNPACK_SEQUENCE_TWO_TUPLE_r12] = _UNPACK_SEQUENCE_TWO_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02] = _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12] = _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23] = _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03] = _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13] = _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE, [_UNPACK_SEQUENCE_TUPLE_r10] = _UNPACK_SEQUENCE_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_TUPLE_r10] = _UNPACK_SEQUENCE_UNIQUE_TUPLE, [_UNPACK_SEQUENCE_LIST_r10] = _UNPACK_SEQUENCE_LIST, [_UNPACK_EX_r10] = _UNPACK_EX, [_STORE_ATTR_r20] = _STORE_ATTR, @@ -3907,13 +4245,13 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_BUILD_TEMPLATE_r21] = _BUILD_TEMPLATE, [_BUILD_TUPLE_r01] = _BUILD_TUPLE, [_BUILD_LIST_r01] = _BUILD_LIST, - [_LIST_EXTEND_r10] = _LIST_EXTEND, + [_LIST_EXTEND_r11] = _LIST_EXTEND, [_SET_UPDATE_r11] = _SET_UPDATE, [_BUILD_SET_r01] = _BUILD_SET, [_BUILD_MAP_r01] = _BUILD_MAP, [_SETUP_ANNOTATIONS_r00] = _SETUP_ANNOTATIONS, - [_DICT_UPDATE_r10] = _DICT_UPDATE, - [_DICT_MERGE_r10] = _DICT_MERGE, + [_DICT_UPDATE_r11] = _DICT_UPDATE, + [_DICT_MERGE_r11] = _DICT_MERGE, [_MAP_ADD_r20] = _MAP_ADD, [_LOAD_SUPER_ATTR_ATTR_r31] = _LOAD_SUPER_ATTR_ATTR, [_LOAD_SUPER_ATTR_METHOD_r32] = _LOAD_SUPER_ATTR_METHOD, @@ -3994,7 +4332,6 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_MATCH_SEQUENCE_r23] = _MATCH_SEQUENCE, [_MATCH_KEYS_r23] = _MATCH_KEYS, [_GET_ITER_r12] = _GET_ITER, - [_GET_YIELD_FROM_ITER_r11] = _GET_YIELD_FROM_ITER, [_FOR_ITER_TIER_TWO_r23] = _FOR_ITER_TIER_TWO, [_ITER_CHECK_LIST_r02] = _ITER_CHECK_LIST, [_ITER_CHECK_LIST_r12] = _ITER_CHECK_LIST, @@ -4123,8 +4460,11 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_CREATE_INIT_FRAME_r01] = _CREATE_INIT_FRAME, [_EXIT_INIT_CHECK_r10] = _EXIT_INIT_CHECK, [_CALL_BUILTIN_CLASS_r01] = _CALL_BUILTIN_CLASS, + [_GUARD_CALLABLE_BUILTIN_O_r00] = _GUARD_CALLABLE_BUILTIN_O, [_CALL_BUILTIN_O_r03] = _CALL_BUILTIN_O, + [_GUARD_CALLABLE_BUILTIN_FAST_r00] = _GUARD_CALLABLE_BUILTIN_FAST, [_CALL_BUILTIN_FAST_r01] = _CALL_BUILTIN_FAST, + [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00] = _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, [_CALL_BUILTIN_FAST_WITH_KEYWORDS_r01] = _CALL_BUILTIN_FAST_WITH_KEYWORDS, [_GUARD_CALLABLE_LEN_r03] = _GUARD_CALLABLE_LEN, [_GUARD_CALLABLE_LEN_r13] = _GUARD_CALLABLE_LEN, @@ -4144,10 +4484,22 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_CALL_LIST_APPEND_r13] = _CALL_LIST_APPEND, [_CALL_LIST_APPEND_r23] = _CALL_LIST_APPEND, [_CALL_LIST_APPEND_r33] = _CALL_LIST_APPEND, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_O, [_CALL_METHOD_DESCRIPTOR_O_r03] = _CALL_METHOD_DESCRIPTOR_O, + [_CHECK_RECURSION_LIMIT_r00] = _CHECK_RECURSION_LIMIT, + [_CHECK_RECURSION_LIMIT_r11] = _CHECK_RECURSION_LIMIT, + [_CHECK_RECURSION_LIMIT_r22] = _CHECK_RECURSION_LIMIT, + [_CHECK_RECURSION_LIMIT_r33] = _CHECK_RECURSION_LIMIT, + [_CALL_METHOD_DESCRIPTOR_O_INLINE_r03] = _CALL_METHOD_DESCRIPTOR_O_INLINE, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01] = _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r01] = _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, [_CALL_METHOD_DESCRIPTOR_NOARGS_r01] = _CALL_METHOD_DESCRIPTOR_NOARGS, + [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r01] = _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, [_CALL_METHOD_DESCRIPTOR_FAST_r01] = _CALL_METHOD_DESCRIPTOR_FAST, + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r01] = _CALL_METHOD_DESCRIPTOR_FAST_INLINE, [_MAYBE_EXPAND_METHOD_KW_r11] = _MAYBE_EXPAND_METHOD_KW, [_PY_FRAME_KW_r11] = _PY_FRAME_KW, [_CHECK_FUNCTION_VERSION_KW_r11] = _CHECK_FUNCTION_VERSION_KW, @@ -4405,10 +4757,26 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_BINARY_OP_ADD_FLOAT_r03] = "_BINARY_OP_ADD_FLOAT_r03", [_BINARY_OP_ADD_FLOAT_r13] = "_BINARY_OP_ADD_FLOAT_r13", [_BINARY_OP_ADD_FLOAT_r23] = "_BINARY_OP_ADD_FLOAT_r23", + [_BINARY_OP_ADD_FLOAT_INPLACE] = "_BINARY_OP_ADD_FLOAT_INPLACE", + [_BINARY_OP_ADD_FLOAT_INPLACE_r03] = "_BINARY_OP_ADD_FLOAT_INPLACE_r03", + [_BINARY_OP_ADD_FLOAT_INPLACE_r13] = "_BINARY_OP_ADD_FLOAT_INPLACE_r13", + [_BINARY_OP_ADD_FLOAT_INPLACE_r23] = "_BINARY_OP_ADD_FLOAT_INPLACE_r23", + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT] = "_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT", + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03] = "_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03", + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13] = "_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13", + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23] = "_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", [_BINARY_OP_ADD_INT_r03] = "_BINARY_OP_ADD_INT_r03", [_BINARY_OP_ADD_INT_r13] = "_BINARY_OP_ADD_INT_r13", [_BINARY_OP_ADD_INT_r23] = "_BINARY_OP_ADD_INT_r23", + [_BINARY_OP_ADD_INT_INPLACE] = "_BINARY_OP_ADD_INT_INPLACE", + [_BINARY_OP_ADD_INT_INPLACE_r03] = "_BINARY_OP_ADD_INT_INPLACE_r03", + [_BINARY_OP_ADD_INT_INPLACE_r13] = "_BINARY_OP_ADD_INT_INPLACE_r13", + [_BINARY_OP_ADD_INT_INPLACE_r23] = "_BINARY_OP_ADD_INT_INPLACE_r23", + [_BINARY_OP_ADD_INT_INPLACE_RIGHT] = "_BINARY_OP_ADD_INT_INPLACE_RIGHT", + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r03] = "_BINARY_OP_ADD_INT_INPLACE_RIGHT_r03", + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r13] = "_BINARY_OP_ADD_INT_INPLACE_RIGHT_r13", + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r23] = "_BINARY_OP_ADD_INT_INPLACE_RIGHT_r23", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", [_BINARY_OP_ADD_UNICODE_r03] = "_BINARY_OP_ADD_UNICODE_r03", [_BINARY_OP_ADD_UNICODE_r13] = "_BINARY_OP_ADD_UNICODE_r13", @@ -4421,14 +4789,32 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_BINARY_OP_MULTIPLY_FLOAT_r03] = "_BINARY_OP_MULTIPLY_FLOAT_r03", [_BINARY_OP_MULTIPLY_FLOAT_r13] = "_BINARY_OP_MULTIPLY_FLOAT_r13", [_BINARY_OP_MULTIPLY_FLOAT_r23] = "_BINARY_OP_MULTIPLY_FLOAT_r23", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23", [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", [_BINARY_OP_MULTIPLY_INT_r03] = "_BINARY_OP_MULTIPLY_INT_r03", [_BINARY_OP_MULTIPLY_INT_r13] = "_BINARY_OP_MULTIPLY_INT_r13", [_BINARY_OP_MULTIPLY_INT_r23] = "_BINARY_OP_MULTIPLY_INT_r23", + [_BINARY_OP_MULTIPLY_INT_INPLACE] = "_BINARY_OP_MULTIPLY_INT_INPLACE", + [_BINARY_OP_MULTIPLY_INT_INPLACE_r03] = "_BINARY_OP_MULTIPLY_INT_INPLACE_r03", + [_BINARY_OP_MULTIPLY_INT_INPLACE_r13] = "_BINARY_OP_MULTIPLY_INT_INPLACE_r13", + [_BINARY_OP_MULTIPLY_INT_INPLACE_r23] = "_BINARY_OP_MULTIPLY_INT_INPLACE_r23", + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT] = "_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT", + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03] = "_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03", + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13] = "_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13", + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23] = "_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23", [_BINARY_OP_SUBSCR_CHECK_FUNC] = "_BINARY_OP_SUBSCR_CHECK_FUNC", [_BINARY_OP_SUBSCR_CHECK_FUNC_r23] = "_BINARY_OP_SUBSCR_CHECK_FUNC_r23", [_BINARY_OP_SUBSCR_DICT] = "_BINARY_OP_SUBSCR_DICT", [_BINARY_OP_SUBSCR_DICT_r23] = "_BINARY_OP_SUBSCR_DICT_r23", + [_BINARY_OP_SUBSCR_DICT_KNOWN_HASH] = "_BINARY_OP_SUBSCR_DICT_KNOWN_HASH", + [_BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23] = "_BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23", [_BINARY_OP_SUBSCR_INIT_CALL] = "_BINARY_OP_SUBSCR_INIT_CALL", [_BINARY_OP_SUBSCR_INIT_CALL_r01] = "_BINARY_OP_SUBSCR_INIT_CALL_r01", [_BINARY_OP_SUBSCR_INIT_CALL_r11] = "_BINARY_OP_SUBSCR_INIT_CALL_r11", @@ -4450,10 +4836,26 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_BINARY_OP_SUBTRACT_FLOAT_r03] = "_BINARY_OP_SUBTRACT_FLOAT_r03", [_BINARY_OP_SUBTRACT_FLOAT_r13] = "_BINARY_OP_SUBTRACT_FLOAT_r13", [_BINARY_OP_SUBTRACT_FLOAT_r23] = "_BINARY_OP_SUBTRACT_FLOAT_r23", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23", [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", [_BINARY_OP_SUBTRACT_INT_r03] = "_BINARY_OP_SUBTRACT_INT_r03", [_BINARY_OP_SUBTRACT_INT_r13] = "_BINARY_OP_SUBTRACT_INT_r13", [_BINARY_OP_SUBTRACT_INT_r23] = "_BINARY_OP_SUBTRACT_INT_r23", + [_BINARY_OP_SUBTRACT_INT_INPLACE] = "_BINARY_OP_SUBTRACT_INT_INPLACE", + [_BINARY_OP_SUBTRACT_INT_INPLACE_r03] = "_BINARY_OP_SUBTRACT_INT_INPLACE_r03", + [_BINARY_OP_SUBTRACT_INT_INPLACE_r13] = "_BINARY_OP_SUBTRACT_INT_INPLACE_r13", + [_BINARY_OP_SUBTRACT_INT_INPLACE_r23] = "_BINARY_OP_SUBTRACT_INT_INPLACE_r23", + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT] = "_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT", + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03] = "_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03", + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13] = "_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13", + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23] = "_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23", [_BINARY_SLICE] = "_BINARY_SLICE", [_BINARY_SLICE_r31] = "_BINARY_SLICE_r31", [_BUILD_INTERPOLATION] = "_BUILD_INTERPOLATION", @@ -4485,7 +4887,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_CALL_INTRINSIC_1] = "_CALL_INTRINSIC_1", [_CALL_INTRINSIC_1_r12] = "_CALL_INTRINSIC_1_r12", [_CALL_INTRINSIC_2] = "_CALL_INTRINSIC_2", - [_CALL_INTRINSIC_2_r21] = "_CALL_INTRINSIC_2_r21", + [_CALL_INTRINSIC_2_r23] = "_CALL_INTRINSIC_2_r23", [_CALL_ISINSTANCE] = "_CALL_ISINSTANCE", [_CALL_ISINSTANCE_r31] = "_CALL_ISINSTANCE_r31", [_CALL_KW_NON_PY] = "_CALL_KW_NON_PY", @@ -4499,12 +4901,20 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_CALL_LIST_APPEND_r33] = "_CALL_LIST_APPEND_r33", [_CALL_METHOD_DESCRIPTOR_FAST] = "_CALL_METHOD_DESCRIPTOR_FAST", [_CALL_METHOD_DESCRIPTOR_FAST_r01] = "_CALL_METHOD_DESCRIPTOR_FAST_r01", + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = "_CALL_METHOD_DESCRIPTOR_FAST_INLINE", + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r01] = "_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r01", [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01", + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE", + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r01] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r01", [_CALL_METHOD_DESCRIPTOR_NOARGS] = "_CALL_METHOD_DESCRIPTOR_NOARGS", [_CALL_METHOD_DESCRIPTOR_NOARGS_r01] = "_CALL_METHOD_DESCRIPTOR_NOARGS_r01", + [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE] = "_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE", + [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r01] = "_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r01", [_CALL_METHOD_DESCRIPTOR_O] = "_CALL_METHOD_DESCRIPTOR_O", [_CALL_METHOD_DESCRIPTOR_O_r03] = "_CALL_METHOD_DESCRIPTOR_O_r03", + [_CALL_METHOD_DESCRIPTOR_O_INLINE] = "_CALL_METHOD_DESCRIPTOR_O_INLINE", + [_CALL_METHOD_DESCRIPTOR_O_INLINE_r03] = "_CALL_METHOD_DESCRIPTOR_O_INLINE_r03", [_CALL_NON_PY_GENERAL] = "_CALL_NON_PY_GENERAL", [_CALL_NON_PY_GENERAL_r01] = "_CALL_NON_PY_GENERAL_r01", [_CALL_STR_1] = "_CALL_STR_1", @@ -4577,6 +4987,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_CHECK_PERIODIC_r00] = "_CHECK_PERIODIC_r00", [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = "_CHECK_PERIODIC_IF_NOT_YIELD_FROM", [_CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00] = "_CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00", + [_CHECK_RECURSION_LIMIT] = "_CHECK_RECURSION_LIMIT", + [_CHECK_RECURSION_LIMIT_r00] = "_CHECK_RECURSION_LIMIT_r00", + [_CHECK_RECURSION_LIMIT_r11] = "_CHECK_RECURSION_LIMIT_r11", + [_CHECK_RECURSION_LIMIT_r22] = "_CHECK_RECURSION_LIMIT_r22", + [_CHECK_RECURSION_LIMIT_r33] = "_CHECK_RECURSION_LIMIT_r33", [_CHECK_RECURSION_REMAINING] = "_CHECK_RECURSION_REMAINING", [_CHECK_RECURSION_REMAINING_r00] = "_CHECK_RECURSION_REMAINING_r00", [_CHECK_RECURSION_REMAINING_r11] = "_CHECK_RECURSION_REMAINING_r11", @@ -4656,9 +5071,9 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_DEOPT_r20] = "_DEOPT_r20", [_DEOPT_r30] = "_DEOPT_r30", [_DICT_MERGE] = "_DICT_MERGE", - [_DICT_MERGE_r10] = "_DICT_MERGE_r10", + [_DICT_MERGE_r11] = "_DICT_MERGE_r11", [_DICT_UPDATE] = "_DICT_UPDATE", - [_DICT_UPDATE_r10] = "_DICT_UPDATE_r10", + [_DICT_UPDATE_r11] = "_DICT_UPDATE_r11", [_DYNAMIC_EXIT] = "_DYNAMIC_EXIT", [_DYNAMIC_EXIT_r00] = "_DYNAMIC_EXIT_r00", [_DYNAMIC_EXIT_r10] = "_DYNAMIC_EXIT_r10", @@ -4667,7 +5082,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_END_FOR] = "_END_FOR", [_END_FOR_r10] = "_END_FOR_r10", [_END_SEND] = "_END_SEND", - [_END_SEND_r21] = "_END_SEND_r21", + [_END_SEND_r31] = "_END_SEND_r31", [_ERROR_POP_N] = "_ERROR_POP_N", [_ERROR_POP_N_r00] = "_ERROR_POP_N_r00", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", @@ -4706,8 +5121,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GET_ITER_r12] = "_GET_ITER_r12", [_GET_LEN] = "_GET_LEN", [_GET_LEN_r12] = "_GET_LEN_r12", - [_GET_YIELD_FROM_ITER] = "_GET_YIELD_FROM_ITER", - [_GET_YIELD_FROM_ITER_r11] = "_GET_YIELD_FROM_ITER_r11", [_GUARD_BINARY_OP_EXTEND] = "_GUARD_BINARY_OP_EXTEND", [_GUARD_BINARY_OP_EXTEND_r22] = "_GUARD_BINARY_OP_EXTEND_r22", [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS", @@ -4765,6 +5178,12 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_BIT_IS_UNSET_POP_7_r10] = "_GUARD_BIT_IS_UNSET_POP_7_r10", [_GUARD_BIT_IS_UNSET_POP_7_r21] = "_GUARD_BIT_IS_UNSET_POP_7_r21", [_GUARD_BIT_IS_UNSET_POP_7_r32] = "_GUARD_BIT_IS_UNSET_POP_7_r32", + [_GUARD_CALLABLE_BUILTIN_FAST] = "_GUARD_CALLABLE_BUILTIN_FAST", + [_GUARD_CALLABLE_BUILTIN_FAST_r00] = "_GUARD_CALLABLE_BUILTIN_FAST_r00", + [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS] = "_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS", + [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00] = "_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00", + [_GUARD_CALLABLE_BUILTIN_O] = "_GUARD_CALLABLE_BUILTIN_O", + [_GUARD_CALLABLE_BUILTIN_O_r00] = "_GUARD_CALLABLE_BUILTIN_O_r00", [_GUARD_CALLABLE_ISINSTANCE] = "_GUARD_CALLABLE_ISINSTANCE", [_GUARD_CALLABLE_ISINSTANCE_r03] = "_GUARD_CALLABLE_ISINSTANCE_r03", [_GUARD_CALLABLE_ISINSTANCE_r13] = "_GUARD_CALLABLE_ISINSTANCE_r13", @@ -4780,6 +5199,14 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_CALLABLE_LIST_APPEND_r13] = "_GUARD_CALLABLE_LIST_APPEND_r13", [_GUARD_CALLABLE_LIST_APPEND_r23] = "_GUARD_CALLABLE_LIST_APPEND_r23", [_GUARD_CALLABLE_LIST_APPEND_r33] = "_GUARD_CALLABLE_LIST_APPEND_r33", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_O] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_O", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00", [_GUARD_CALLABLE_STR_1] = "_GUARD_CALLABLE_STR_1", [_GUARD_CALLABLE_STR_1_r03] = "_GUARD_CALLABLE_STR_1_r03", [_GUARD_CALLABLE_STR_1_r13] = "_GUARD_CALLABLE_STR_1_r13", @@ -5095,7 +5522,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_LIST_APPEND] = "_LIST_APPEND", [_LIST_APPEND_r10] = "_LIST_APPEND_r10", [_LIST_EXTEND] = "_LIST_EXTEND", - [_LIST_EXTEND_r10] = "_LIST_EXTEND_r10", + [_LIST_EXTEND_r11] = "_LIST_EXTEND_r11", [_LOAD_ATTR] = "_LOAD_ATTR", [_LOAD_ATTR_r10] = "_LOAD_ATTR_r10", [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", @@ -5386,6 +5813,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_PY_FRAME_GENERAL_r01] = "_PY_FRAME_GENERAL_r01", [_PY_FRAME_KW] = "_PY_FRAME_KW", [_PY_FRAME_KW_r11] = "_PY_FRAME_KW_r11", + [_RECORD_3OS_GEN_FUNC] = "_RECORD_3OS_GEN_FUNC", [_RECORD_4OS] = "_RECORD_4OS", [_RECORD_BOUND_METHOD] = "_RECORD_BOUND_METHOD", [_RECORD_CALLABLE] = "_RECORD_CALLABLE", @@ -5413,7 +5841,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_SAVE_RETURN_OFFSET_r22] = "_SAVE_RETURN_OFFSET_r22", [_SAVE_RETURN_OFFSET_r33] = "_SAVE_RETURN_OFFSET_r33", [_SEND_GEN_FRAME] = "_SEND_GEN_FRAME", - [_SEND_GEN_FRAME_r22] = "_SEND_GEN_FRAME_r22", + [_SEND_GEN_FRAME_r33] = "_SEND_GEN_FRAME_r33", [_SETUP_ANNOTATIONS] = "_SETUP_ANNOTATIONS", [_SETUP_ANNOTATIONS_r00] = "_SETUP_ANNOTATIONS_r00", [_SET_ADD] = "_SET_ADD", @@ -5475,6 +5903,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_STORE_SUBSCR_r30] = "_STORE_SUBSCR_r30", [_STORE_SUBSCR_DICT] = "_STORE_SUBSCR_DICT", [_STORE_SUBSCR_DICT_r31] = "_STORE_SUBSCR_DICT_r31", + [_STORE_SUBSCR_DICT_KNOWN_HASH] = "_STORE_SUBSCR_DICT_KNOWN_HASH", + [_STORE_SUBSCR_DICT_KNOWN_HASH_r31] = "_STORE_SUBSCR_DICT_KNOWN_HASH_r31", [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", [_STORE_SUBSCR_LIST_INT_r32] = "_STORE_SUBSCR_LIST_INT_r32", [_SWAP] = "_SWAP", @@ -5567,6 +5997,10 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_UNARY_INVERT_r12] = "_UNARY_INVERT_r12", [_UNARY_NEGATIVE] = "_UNARY_NEGATIVE", [_UNARY_NEGATIVE_r12] = "_UNARY_NEGATIVE_r12", + [_UNARY_NEGATIVE_FLOAT_INPLACE] = "_UNARY_NEGATIVE_FLOAT_INPLACE", + [_UNARY_NEGATIVE_FLOAT_INPLACE_r02] = "_UNARY_NEGATIVE_FLOAT_INPLACE_r02", + [_UNARY_NEGATIVE_FLOAT_INPLACE_r12] = "_UNARY_NEGATIVE_FLOAT_INPLACE_r12", + [_UNARY_NEGATIVE_FLOAT_INPLACE_r23] = "_UNARY_NEGATIVE_FLOAT_INPLACE_r23", [_UNARY_NOT] = "_UNARY_NOT", [_UNARY_NOT_r01] = "_UNARY_NOT_r01", [_UNARY_NOT_r11] = "_UNARY_NOT_r11", @@ -5582,6 +6016,15 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_UNPACK_SEQUENCE_TUPLE_r10] = "_UNPACK_SEQUENCE_TUPLE_r10", [_UNPACK_SEQUENCE_TWO_TUPLE] = "_UNPACK_SEQUENCE_TWO_TUPLE", [_UNPACK_SEQUENCE_TWO_TUPLE_r12] = "_UNPACK_SEQUENCE_TWO_TUPLE_r12", + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE] = "_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE", + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03] = "_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03", + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13] = "_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13", + [_UNPACK_SEQUENCE_UNIQUE_TUPLE] = "_UNPACK_SEQUENCE_UNIQUE_TUPLE", + [_UNPACK_SEQUENCE_UNIQUE_TUPLE_r10] = "_UNPACK_SEQUENCE_UNIQUE_TUPLE_r10", + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE] = "_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE", + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02] = "_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02", + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12] = "_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12", + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23] = "_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23", [_WITH_EXCEPT_START] = "_WITH_EXCEPT_START", [_WITH_EXCEPT_START_r33] = "_WITH_EXCEPT_START_r33", [_YIELD_VALUE] = "_YIELD_VALUE", @@ -5687,9 +6130,11 @@ int _PyUop_num_popped(int opcode, int oparg) case _POP_ITER: return 2; case _END_SEND: - return 2; + return 3; case _UNARY_NEGATIVE: return 1; + case _UNARY_NEGATIVE_FLOAT_INPLACE: + return 1; case _UNARY_NOT: return 1; case _TO_BOOL: @@ -5734,6 +6179,18 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_SUBTRACT_INT: return 2; + case _BINARY_OP_ADD_INT_INPLACE: + return 2; + case _BINARY_OP_SUBTRACT_INT_INPLACE: + return 2; + case _BINARY_OP_MULTIPLY_INT_INPLACE: + return 2; + case _BINARY_OP_ADD_INT_INPLACE_RIGHT: + return 2; + case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT: + return 2; + case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT: + return 2; case _GUARD_NOS_FLOAT: return 0; case _GUARD_TOS_FLOAT: @@ -5744,6 +6201,18 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_SUBTRACT_FLOAT: return 2; + case _BINARY_OP_ADD_FLOAT_INPLACE: + return 2; + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE: + return 2; + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE: + return 2; + case _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT: + return 2; + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT: + return 2; + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT: + return 2; case _BINARY_OP_ADD_UNICODE: return 2; case _BINARY_OP_INPLACE_ADD_UNICODE: @@ -5782,6 +6251,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GUARD_TOS_FROZENDICT: return 0; + case _BINARY_OP_SUBSCR_DICT_KNOWN_HASH: + return 2; case _BINARY_OP_SUBSCR_DICT: return 2; case _BINARY_OP_SUBSCR_CHECK_FUNC: @@ -5798,6 +6269,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 3; case _STORE_SUBSCR_DICT: return 3; + case _STORE_SUBSCR_DICT_KNOWN_HASH: + return 3; case _DELETE_SUBSCR: return 2; case _CALL_INTRINSIC_1: @@ -5832,8 +6305,14 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _UNPACK_SEQUENCE_TWO_TUPLE: return 1; + case _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE: + return 1; + case _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE: + return 1; case _UNPACK_SEQUENCE_TUPLE: return 1; + case _UNPACK_SEQUENCE_UNIQUE_TUPLE: + return 1; case _UNPACK_SEQUENCE_LIST: return 1; case _UNPACK_EX: @@ -5980,8 +6459,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GET_ITER: return 1; - case _GET_YIELD_FROM_ITER: - return 1; case _FOR_ITER_TIER_TWO: return 0; case _ITER_CHECK_LIST: @@ -6096,10 +6573,16 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _CALL_BUILTIN_CLASS: return 2 + oparg; + case _GUARD_CALLABLE_BUILTIN_O: + return 0; case _CALL_BUILTIN_O: return 2 + oparg; + case _GUARD_CALLABLE_BUILTIN_FAST: + return 0; case _CALL_BUILTIN_FAST: return 2 + oparg; + case _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS: + return 0; case _CALL_BUILTIN_FAST_WITH_KEYWORDS: return 2 + oparg; case _GUARD_CALLABLE_LEN: @@ -6114,14 +6597,32 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _CALL_LIST_APPEND: return 3; + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_O: + return 0; case _CALL_METHOD_DESCRIPTOR_O: return 2 + oparg; + case _CHECK_RECURSION_LIMIT: + return 0; + case _CALL_METHOD_DESCRIPTOR_O_INLINE: + return 1 + oparg; + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: + return 0; case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE: + return 1 + oparg; + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS: + return 0; case _CALL_METHOD_DESCRIPTOR_NOARGS: return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE: + return 1 + oparg; + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST: + return 0; case _CALL_METHOD_DESCRIPTOR_FAST: return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_FAST_INLINE: + return 1 + oparg; case _MAYBE_EXPAND_METHOD_KW: return 0; case _PY_FRAME_KW: @@ -6298,6 +6799,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _RECORD_NOS_GEN_FUNC: return 0; + case _RECORD_3OS_GEN_FUNC: + return 0; case _RECORD_4OS: return 0; case _RECORD_CALLABLE: diff --git a/Include/object.h b/Include/object.h index 3fb28035a50547..10d9d76d93454a 100644 --- a/Include/object.h +++ b/Include/object.h @@ -127,7 +127,7 @@ whose size is determined when the object is allocated. struct _object { _Py_ANONYMOUS union { #if SIZEOF_VOID_P > 4 - PY_INT64_T ob_refcnt_full; /* This field is needed for efficient initialization with Clang on ARM */ + int64_t ob_refcnt_full; /* This field is needed for efficient initialization with Clang on ARM */ struct { # if PY_BIG_ENDIAN uint16_t ob_flags; diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index f9173fd83c295e..ac6d4d964d3b5e 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -26,111 +26,110 @@ extern "C" { #define FORMAT_WITH_SPEC 13 #define GET_AITER 14 #define GET_ANEXT 15 -#define GET_ITER 16 +#define GET_LEN 16 #define RESERVED 17 -#define GET_LEN 18 -#define GET_YIELD_FROM_ITER 19 -#define INTERPRETER_EXIT 20 -#define LOAD_BUILD_CLASS 21 -#define LOAD_LOCALS 22 -#define MAKE_FUNCTION 23 -#define MATCH_KEYS 24 -#define MATCH_MAPPING 25 -#define MATCH_SEQUENCE 26 -#define NOP 27 -#define NOT_TAKEN 28 -#define POP_EXCEPT 29 -#define POP_ITER 30 -#define POP_TOP 31 -#define PUSH_EXC_INFO 32 -#define PUSH_NULL 33 -#define RETURN_GENERATOR 34 -#define RETURN_VALUE 35 -#define SETUP_ANNOTATIONS 36 -#define STORE_SLICE 37 -#define STORE_SUBSCR 38 -#define TO_BOOL 39 -#define UNARY_INVERT 40 -#define UNARY_NEGATIVE 41 -#define UNARY_NOT 42 -#define WITH_EXCEPT_START 43 -#define BINARY_OP 44 -#define BUILD_INTERPOLATION 45 -#define BUILD_LIST 46 -#define BUILD_MAP 47 -#define BUILD_SET 48 -#define BUILD_SLICE 49 -#define BUILD_STRING 50 -#define BUILD_TUPLE 51 -#define CALL 52 -#define CALL_INTRINSIC_1 53 -#define CALL_INTRINSIC_2 54 -#define CALL_KW 55 -#define COMPARE_OP 56 -#define CONTAINS_OP 57 -#define CONVERT_VALUE 58 -#define COPY 59 -#define COPY_FREE_VARS 60 -#define DELETE_ATTR 61 -#define DELETE_DEREF 62 -#define DELETE_FAST 63 -#define DELETE_GLOBAL 64 -#define DELETE_NAME 65 -#define DICT_MERGE 66 -#define DICT_UPDATE 67 -#define END_ASYNC_FOR 68 -#define EXTENDED_ARG 69 -#define FOR_ITER 70 -#define GET_AWAITABLE 71 -#define IMPORT_FROM 72 -#define IMPORT_NAME 73 -#define IS_OP 74 -#define JUMP_BACKWARD 75 -#define JUMP_BACKWARD_NO_INTERRUPT 76 -#define JUMP_FORWARD 77 -#define LIST_APPEND 78 -#define LIST_EXTEND 79 -#define LOAD_ATTR 80 -#define LOAD_COMMON_CONSTANT 81 -#define LOAD_CONST 82 -#define LOAD_DEREF 83 -#define LOAD_FAST 84 -#define LOAD_FAST_AND_CLEAR 85 -#define LOAD_FAST_BORROW 86 -#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 87 -#define LOAD_FAST_CHECK 88 -#define LOAD_FAST_LOAD_FAST 89 -#define LOAD_FROM_DICT_OR_DEREF 90 -#define LOAD_FROM_DICT_OR_GLOBALS 91 -#define LOAD_GLOBAL 92 -#define LOAD_NAME 93 -#define LOAD_SMALL_INT 94 -#define LOAD_SPECIAL 95 -#define LOAD_SUPER_ATTR 96 -#define MAKE_CELL 97 -#define MAP_ADD 98 -#define MATCH_CLASS 99 -#define POP_JUMP_IF_FALSE 100 -#define POP_JUMP_IF_NONE 101 -#define POP_JUMP_IF_NOT_NONE 102 -#define POP_JUMP_IF_TRUE 103 -#define RAISE_VARARGS 104 -#define RERAISE 105 -#define SEND 106 -#define SET_ADD 107 -#define SET_FUNCTION_ATTRIBUTE 108 -#define SET_UPDATE 109 -#define STORE_ATTR 110 -#define STORE_DEREF 111 -#define STORE_FAST 112 -#define STORE_FAST_LOAD_FAST 113 -#define STORE_FAST_STORE_FAST 114 -#define STORE_GLOBAL 115 -#define STORE_NAME 116 -#define SWAP 117 -#define UNPACK_EX 118 -#define UNPACK_SEQUENCE 119 -#define YIELD_VALUE 120 +#define INTERPRETER_EXIT 18 +#define LOAD_BUILD_CLASS 19 +#define LOAD_LOCALS 20 +#define MAKE_FUNCTION 21 +#define MATCH_KEYS 22 +#define MATCH_MAPPING 23 +#define MATCH_SEQUENCE 24 +#define NOP 25 +#define NOT_TAKEN 26 +#define POP_EXCEPT 27 +#define POP_ITER 28 +#define POP_TOP 29 +#define PUSH_EXC_INFO 30 +#define PUSH_NULL 31 +#define RETURN_GENERATOR 32 +#define RETURN_VALUE 33 +#define SETUP_ANNOTATIONS 34 +#define STORE_SLICE 35 +#define STORE_SUBSCR 36 +#define TO_BOOL 37 +#define UNARY_INVERT 38 +#define UNARY_NEGATIVE 39 +#define UNARY_NOT 40 +#define WITH_EXCEPT_START 41 +#define BINARY_OP 42 +#define BUILD_INTERPOLATION 43 +#define BUILD_LIST 44 +#define BUILD_MAP 45 +#define BUILD_SET 46 +#define BUILD_SLICE 47 +#define BUILD_STRING 48 +#define BUILD_TUPLE 49 +#define CALL 50 +#define CALL_INTRINSIC_1 51 +#define CALL_INTRINSIC_2 52 +#define CALL_KW 53 +#define COMPARE_OP 54 +#define CONTAINS_OP 55 +#define CONVERT_VALUE 56 +#define COPY 57 +#define COPY_FREE_VARS 58 +#define DELETE_ATTR 59 +#define DELETE_DEREF 60 +#define DELETE_FAST 61 +#define DELETE_GLOBAL 62 +#define DELETE_NAME 63 +#define DICT_MERGE 64 +#define DICT_UPDATE 65 +#define END_ASYNC_FOR 66 +#define EXTENDED_ARG 67 +#define FOR_ITER 68 +#define GET_AWAITABLE 69 +#define GET_ITER 70 +#define IMPORT_FROM 71 +#define IMPORT_NAME 72 +#define IS_OP 73 +#define JUMP_BACKWARD 74 +#define JUMP_BACKWARD_NO_INTERRUPT 75 +#define JUMP_FORWARD 76 +#define LIST_APPEND 77 +#define LIST_EXTEND 78 +#define LOAD_ATTR 79 +#define LOAD_COMMON_CONSTANT 80 +#define LOAD_CONST 81 +#define LOAD_DEREF 82 +#define LOAD_FAST 83 +#define LOAD_FAST_AND_CLEAR 84 +#define LOAD_FAST_BORROW 85 +#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 86 +#define LOAD_FAST_CHECK 87 +#define LOAD_FAST_LOAD_FAST 88 +#define LOAD_FROM_DICT_OR_DEREF 89 +#define LOAD_FROM_DICT_OR_GLOBALS 90 +#define LOAD_GLOBAL 91 +#define LOAD_NAME 92 +#define LOAD_SMALL_INT 93 +#define LOAD_SPECIAL 94 +#define LOAD_SUPER_ATTR 95 +#define MAKE_CELL 96 +#define MAP_ADD 97 +#define MATCH_CLASS 98 +#define POP_JUMP_IF_FALSE 99 +#define POP_JUMP_IF_NONE 100 +#define POP_JUMP_IF_NOT_NONE 101 +#define POP_JUMP_IF_TRUE 102 +#define RAISE_VARARGS 103 +#define RERAISE 104 +#define SEND 105 +#define SET_ADD 106 +#define SET_FUNCTION_ATTRIBUTE 107 +#define SET_UPDATE 108 +#define STORE_ATTR 109 +#define STORE_DEREF 110 +#define STORE_FAST 111 +#define STORE_FAST_LOAD_FAST 112 +#define STORE_FAST_STORE_FAST 113 +#define STORE_GLOBAL 114 +#define STORE_NAME 115 +#define SWAP 116 +#define UNPACK_EX 117 +#define UNPACK_SEQUENCE 118 +#define YIELD_VALUE 119 #define RESUME 128 #define BINARY_OP_ADD_FLOAT 129 #define BINARY_OP_ADD_INT 130 @@ -252,7 +251,7 @@ extern "C" { #define SETUP_WITH 265 #define STORE_FAST_MAYBE_NULL 266 -#define HAVE_ARGUMENT 43 +#define HAVE_ARGUMENT 41 #define MIN_SPECIALIZED_OPCODE 129 #define MIN_INSTRUMENTED_OPCODE 233 diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 7cffd74125f1b4..154bdb0721d3d1 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -61,4 +61,32 @@ #define PYTHON_ABI_VERSION 3 #define PYTHON_ABI_STRING "3" + +/* Stable ABI for free-threaded builds (introduced in PEP 803) + is enabled by one of: + - Py_TARGET_ABI3T, or + - Py_LIMITED_API and Py_GIL_DISABLED. + "Output" macros to be used internally: + - Py_LIMITED_API (defines the subset of API we expose) + - _Py_OPAQUE_PYOBJECT (additionally hides what's ABI-incompatible between + free-threaded & GIL) + (Don't use Py_TARGET_ABI3T directly: it's currently only used to set these + 2 macros. It's also available for users' convenience.) + */ +#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) \ + && !defined(Py_TARGET_ABI3T) +# define Py_TARGET_ABI3T Py_LIMITED_API +#endif +#if defined(Py_TARGET_ABI3T) +# define _Py_OPAQUE_PYOBJECT +# if !defined(Py_LIMITED_API) +# define Py_LIMITED_API Py_TARGET_ABI3T +# elif Py_LIMITED_API > Py_TARGET_ABI3T + // if both are defined, use the *lower* version, + // i.e. maximum compatibility +# undef Py_LIMITED_API +# define Py_LIMITED_API Py_TARGET_ABI3T +# endif +#endif + #endif //_Py_PATCHLEVEL_H diff --git a/Include/pyport.h b/Include/pyport.h index 1e1702abd99a2c..f7bb5d513b9ae6 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -446,7 +446,9 @@ extern "C" { /* * Specify alignment on compilers that support it. */ -#if defined(__GNUC__) && __GNUC__ >= 3 +#ifdef Py_BUILD_CORE +// always use _Py_ALIGNED_DEF instead +#elif defined(__GNUC__) && __GNUC__ >= 3 #define Py_ALIGNED(x) __attribute__((aligned(x))) #else #define Py_ALIGNED(x) diff --git a/Include/refcount.h b/Include/refcount.h index 51346c7e519321..bcdabad3dcb4ff 100644 --- a/Include/refcount.h +++ b/Include/refcount.h @@ -126,7 +126,7 @@ static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) return (_Py_atomic_load_uint32_relaxed(&op->ob_ref_local) == _Py_IMMORTAL_REFCNT_LOCAL); #elif SIZEOF_VOID_P > 4 - return _Py_CAST(PY_INT32_T, op->ob_refcnt) < 0; + return _Py_CAST(int32_t, op->ob_refcnt) < 0; #else return op->ob_refcnt >= _Py_IMMORTAL_MINIMUM_REFCNT; #endif @@ -164,7 +164,7 @@ static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { } #ifndef Py_GIL_DISABLED #if SIZEOF_VOID_P > 4 - ob->ob_refcnt = (PY_UINT32_T)refcnt; + ob->ob_refcnt = (uint32_t)refcnt; #else ob->ob_refcnt = refcnt; #endif @@ -278,7 +278,7 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) _Py_atomic_add_ssize(&op->ob_ref_shared, (1 << _Py_REF_SHARED_SHIFT)); } #elif SIZEOF_VOID_P > 4 - PY_UINT32_T cur_refcnt = op->ob_refcnt; + uint32_t cur_refcnt = op->ob_refcnt; if (cur_refcnt >= _Py_IMMORTAL_INITIAL_REFCNT) { // the object is immortal _Py_INCREF_IMMORTAL_STAT_INC(); @@ -387,7 +387,7 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) #if SIZEOF_VOID_P > 4 /* If an object has been freed, it will have a negative full refcnt * If it has not it been freed, will have a very large refcnt */ - if (op->ob_refcnt_full <= 0 || op->ob_refcnt > (((PY_UINT32_T)-1) - (1<<20))) { + if (op->ob_refcnt_full <= 0 || op->ob_refcnt > (((uint32_t)-1) - (1<<20))) { #else if (op->ob_refcnt <= 0) { #endif diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index b72d581ec25804..29f1d1b01c161f 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -64,13 +64,9 @@ Copyright (c) Corporation for National Research Initiatives. #error Must define SIZEOF_WCHAR_T #endif +/* Soft-deprecated defines */ #define Py_UNICODE_SIZE SIZEOF_WCHAR_T - -/* If wchar_t can be used for UCS-4 storage, set Py_UNICODE_WIDE. - Otherwise, Unicode strings are stored as UCS-2 (with limited support - for UTF-16) */ - -#if Py_UNICODE_SIZE >= 4 +#if SIZEOF_WCHAR_T >= 4 #define Py_UNICODE_WIDE #endif diff --git a/InternalDocs/interpreter.md b/InternalDocs/interpreter.md index 75acdf596a7f30..7fc41a807dd566 100644 --- a/InternalDocs/interpreter.md +++ b/InternalDocs/interpreter.md @@ -507,6 +507,38 @@ After the last `DEOPT_IF` has passed, a hit should be recorded with After an optimization has been deferred in the adaptive instruction, that should be recorded with `STAT_INC(BASE_INSTRUCTION, deferred)`. +## Interpreter types +There are three different types of interpreters to choose from based on compiler support: + + * traditional switch-case interpreter + + Supported by all compilers covered in PEP 7. + + * computed-gotos interpreter + + Enabled using configure option `--with-computed-gotos` and used by default on supported compilers. + It uses [Labels as Values](https://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html) + for more efficient dispatching. + + * tail-calling interpreter + + Enabled using configure option `--with-tail-call-interp` (or `--tail-call-interp` for build.bat on Windows). + It uses [tail calls](https://clang.llvm.org/docs/AttributeReference.html#musttail) and the + [preserve_none](https://clang.llvm.org/docs/AttributeReference.html#preserve-none) + calling convention between the small C functions that implement individual Python opcodes. + + Not all compilers support these and if they do not all targets might be supported (for example, + MSVC currently only supports x64 and only in optimized builds). + + In addition, compilers must do [escape analysis](https://gcc.gnu.org/onlinedocs/gcc/Common-Attributes.html#index-musttail) + of the lifetimes of automatic variables, function parameters, and temporaries to ensure proper tail-calls. They + emit a compile error in case of a violation or detection failure. The ability to detect this varies depending on the compiler and + also on the optimization level. Following techniques are particularly helpful to the MSVC compiler in this regard + * [Introducing additional scopes](https://github.com/python/cpython/blob/3908593039bde9d4b591ab09919003ee57418d64/Python/bytecodes.c#L2526) + * [extracting problematic code paths into a separate function](https://github.com/python/cpython/pull/143068/files#diff-729a985b0cb8b431cb291f1edb561bbbfea22e3f8c262451cd83328a0936a342R3724) + * [returning a pointer instead of taking it as an output parameter](https://github.com/python/cpython/blob/3908593039bde9d4b591ab09919003ee57418d64/Include/internal/pycore_ceval.h#L489-L492) + + Using `restrict` is another (currently unused) remedy. Additional resources -------------------- diff --git a/InternalDocs/jit.md b/InternalDocs/jit.md index decfccad2d8d37..1345f2db8b34db 100644 --- a/InternalDocs/jit.md +++ b/InternalDocs/jit.md @@ -11,24 +11,54 @@ and this enables optimizations that span multiple instructions. Historically, the adaptive interpreter was referred to as `tier 1` and the JIT as `tier 2`. You will see remnants of this in the code. -## The Optimizer and Executors +## The Trace Recorder and Executors -The program begins running on the adaptive interpreter, until a `JUMP_BACKWARD` -instruction determines that it is "hot" because the counter in its +There are two interpreters in this section: + 1. Adaptive interpreter (the default behavior) + 2. Trace recording interpreter (enabled on JIT builds) + +The program begins running on the adaptive interpreter, until a `JUMP_BACKWARD` or +`RESUME` instruction determines that it is "hot" because the counter in its [inline cache](interpreter.md#inline-cache-entries) indicates that it executed more than some threshold number of times (see [`backoff_counter_triggers`](../Include/internal/pycore_backoff.h)). -It then calls the function `_PyOptimizer_Optimize()` in +It then calls the function `_PyJit_TryInitializeTracing` in [`Python/optimizer.c`](../Python/optimizer.c), passing it the current -[frame](frames.md) and instruction pointer. `_PyOptimizer_Optimize()` -constructs an object of type -[`_PyExecutorObject`](../Include/internal/pycore_optimizer.h) which implements -an optimized version of the instruction trace beginning at this jump. - -The optimizer determines where the trace ends, and the executor is set up +[frame](frames.md), instruction pointer and state. +The interpreter then switches into "tracing mode" via the macro +`ENTER_TRACING()`. On platforms that support computed goto and tail-calling +interpreters, the dispatch table is swapped out, while other platforms that do +not support either use a single flag in the opcode. +Execution between the normal interpreter and tracing interpreter are +interleaved via this dispatch mechanism. This means that while logically +there are two interpreters, the implementation appears to be a single +interpreter. + +During tracing mode, after each interpreter instruction's `DISPATCH()`, +the interpreter jumps to the `TRACE_RECORD` instruction. This instruction +records the previous instruction executed and also any live values of the next +operation it may require. It then translates the previous instruction to +a sequence of micro-ops using `_PyJit_translate_single_bytecode_to_trace`. +To ensure that the adaptive interpreter instructions +and cache entries are up-to-date, the trace recording interpreter always resets +the adaptive counters of adaptive instructions it sees. +This forces a re-specialization of any new instruction should an instruction +deoptimize. Thus, feeding the trace recorder up-to-date information. +Finally, the `TRACE_RECORD` instruction decides when to stop tracing +using various heuristics. + +Once trace recording concludes, `LEAVE_TRACING()` swaps out the dispatch +table/the opcode flag set earlier by `ENTER_TRACING()` is unset. +`stop_tracing_and_jit()` then calls `_PyOptimizer_Optimize()` which optimizes +the trace and constructs an +[`_PyExecutorObject`](../Include/internal/pycore_optimizer.h). + +JIT execution is set up to either return to the adaptive interpreter and resume execution, or transfer control to another executor (see `_PyExitData` in -Include/internal/pycore_optimizer.h). +Include/internal/pycore_optimizer.h). When resuming to the adaptive interpreter, +a "side exit", generated by an `EXIT_IF` may trigger recording of another trace. +While a "deopt", generated by a `DEOPT_IF`, does not trigger recording. The executor is stored on the [`code object`](code_objects.md) of the frame, in the `co_executors` field which is an array of executors. The start @@ -40,12 +70,7 @@ executor in `co_executors`. The micro-op (abbreviated `uop` to approximate `μop`) optimizer is defined in [`Python/optimizer.c`](../Python/optimizer.c) as `_PyOptimizer_Optimize`. -It translates an instruction trace into a sequence of micro-ops by replacing -each bytecode by an equivalent sequence of micro-ops (see -`_PyOpcode_macro_expansion` in -[pycore_opcode_metadata.h](../Include/internal/pycore_opcode_metadata.h) -which is generated from [`Python/bytecodes.c`](../Python/bytecodes.c)). -The micro-op sequence is then optimized by +It takes a micro-op sequence from the trace recorder and optimizes with `_Py_uop_analyze_and_optimize` in [`Python/optimizer_analysis.c`](../Python/optimizer_analysis.c) and an instance of `_PyUOpExecutor_Type` is created to contain it. diff --git a/InternalDocs/string_interning.md b/InternalDocs/string_interning.md index 26a5197c6e70f3..0913b1a3471ef4 100644 --- a/InternalDocs/string_interning.md +++ b/InternalDocs/string_interning.md @@ -52,15 +52,9 @@ The key and value of each entry in this dict reference the same object. ## Immortality and reference counting -Invariant: Every immortal string is interned. +In the GIL-enabled build interned strings may be mortal or immortal. In the +free-threaded build, interned strings are always immortal. -In practice, this means that you must not use `_Py_SetImmortal` on -a string. (If you know it's already immortal, don't immortalize it; -if you know it's not interned you might be immortalizing a redundant copy; -if it's interned and mortal it needs extra processing in -`_PyUnicode_InternImmortal`.) - -The converse is not true: interned strings can be mortal. For mortal interned strings: - the 2 references from the interned dict (key & value) are excluded from diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 5c4903f14aa86b..8361ddbea89716 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -15,7 +15,6 @@ class ANSIColors: RESET = "\x1b[0m" - BLACK = "\x1b[30m" BLUE = "\x1b[34m" CYAN = "\x1b[36m" @@ -200,6 +199,30 @@ class Difflib(ThemeSection): reset: str = ANSIColors.RESET +@dataclass(frozen=True, kw_only=True) +class FancyCompleter(ThemeSection): + # functions and methods + function: str = ANSIColors.BOLD_BLUE + builtin_function_or_method: str = ANSIColors.BOLD_BLUE + method: str = ANSIColors.BOLD_CYAN + method_wrapper: str = ANSIColors.BOLD_CYAN + wrapper_descriptor: str = ANSIColors.BOLD_CYAN + method_descriptor: str = ANSIColors.BOLD_CYAN + + # numbers + int: str = ANSIColors.BOLD_YELLOW + float: str = ANSIColors.BOLD_YELLOW + complex: str = ANSIColors.BOLD_YELLOW + bool: str = ANSIColors.BOLD_YELLOW + + # others + type: str = ANSIColors.BOLD_MAGENTA + module: str = ANSIColors.CYAN + NoneType: str = ANSIColors.GREY + bytes: str = ANSIColors.BOLD_GREEN + str: str = ANSIColors.BOLD_GREEN + + @dataclass(frozen=True, kw_only=True) class LiveProfiler(ThemeSection): """Theme section for the live profiling TUI (Tachyon profiler). @@ -327,6 +350,7 @@ class Syntax(ThemeSection): class Traceback(ThemeSection): type: str = ANSIColors.BOLD_MAGENTA message: str = ANSIColors.MAGENTA + note: str = ANSIColors.CYAN filename: str = ANSIColors.MAGENTA line_no: str = ANSIColors.MAGENTA frame: str = ANSIColors.MAGENTA @@ -353,6 +377,7 @@ class Theme: """ argparse: Argparse = field(default_factory=Argparse) difflib: Difflib = field(default_factory=Difflib) + fancycompleter: FancyCompleter = field(default_factory=FancyCompleter) live_profiler: LiveProfiler = field(default_factory=LiveProfiler) syntax: Syntax = field(default_factory=Syntax) traceback: Traceback = field(default_factory=Traceback) @@ -363,6 +388,7 @@ def copy_with( *, argparse: Argparse | None = None, difflib: Difflib | None = None, + fancycompleter: FancyCompleter | None = None, live_profiler: LiveProfiler | None = None, syntax: Syntax | None = None, traceback: Traceback | None = None, @@ -376,6 +402,7 @@ def copy_with( return type(self)( argparse=argparse or self.argparse, difflib=difflib or self.difflib, + fancycompleter=fancycompleter or self.fancycompleter, live_profiler=live_profiler or self.live_profiler, syntax=syntax or self.syntax, traceback=traceback or self.traceback, @@ -393,6 +420,7 @@ def no_colors(cls) -> Self: return cls( argparse=Argparse.no_colors(), difflib=Difflib.no_colors(), + fancycompleter=FancyCompleter.no_colors(), live_profiler=LiveProfiler.no_colors(), syntax=Syntax.no_colors(), traceback=Traceback.no_colors(), diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 8d2c1ece8bc6a8..f5954e4372a980 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -235,110 +235,109 @@ FORMAT_WITH_SPEC=13, GET_AITER=14, GET_ANEXT=15, - GET_ITER=16, - GET_LEN=18, - GET_YIELD_FROM_ITER=19, - INTERPRETER_EXIT=20, - LOAD_BUILD_CLASS=21, - LOAD_LOCALS=22, - MAKE_FUNCTION=23, - MATCH_KEYS=24, - MATCH_MAPPING=25, - MATCH_SEQUENCE=26, - NOP=27, - NOT_TAKEN=28, - POP_EXCEPT=29, - POP_ITER=30, - POP_TOP=31, - PUSH_EXC_INFO=32, - PUSH_NULL=33, - RETURN_GENERATOR=34, - RETURN_VALUE=35, - SETUP_ANNOTATIONS=36, - STORE_SLICE=37, - STORE_SUBSCR=38, - TO_BOOL=39, - UNARY_INVERT=40, - UNARY_NEGATIVE=41, - UNARY_NOT=42, - WITH_EXCEPT_START=43, - BINARY_OP=44, - BUILD_INTERPOLATION=45, - BUILD_LIST=46, - BUILD_MAP=47, - BUILD_SET=48, - BUILD_SLICE=49, - BUILD_STRING=50, - BUILD_TUPLE=51, - CALL=52, - CALL_INTRINSIC_1=53, - CALL_INTRINSIC_2=54, - CALL_KW=55, - COMPARE_OP=56, - CONTAINS_OP=57, - CONVERT_VALUE=58, - COPY=59, - COPY_FREE_VARS=60, - DELETE_ATTR=61, - DELETE_DEREF=62, - DELETE_FAST=63, - DELETE_GLOBAL=64, - DELETE_NAME=65, - DICT_MERGE=66, - DICT_UPDATE=67, - END_ASYNC_FOR=68, - EXTENDED_ARG=69, - FOR_ITER=70, - GET_AWAITABLE=71, - IMPORT_FROM=72, - IMPORT_NAME=73, - IS_OP=74, - JUMP_BACKWARD=75, - JUMP_BACKWARD_NO_INTERRUPT=76, - JUMP_FORWARD=77, - LIST_APPEND=78, - LIST_EXTEND=79, - LOAD_ATTR=80, - LOAD_COMMON_CONSTANT=81, - LOAD_CONST=82, - LOAD_DEREF=83, - LOAD_FAST=84, - LOAD_FAST_AND_CLEAR=85, - LOAD_FAST_BORROW=86, - LOAD_FAST_BORROW_LOAD_FAST_BORROW=87, - LOAD_FAST_CHECK=88, - LOAD_FAST_LOAD_FAST=89, - LOAD_FROM_DICT_OR_DEREF=90, - LOAD_FROM_DICT_OR_GLOBALS=91, - LOAD_GLOBAL=92, - LOAD_NAME=93, - LOAD_SMALL_INT=94, - LOAD_SPECIAL=95, - LOAD_SUPER_ATTR=96, - MAKE_CELL=97, - MAP_ADD=98, - MATCH_CLASS=99, - POP_JUMP_IF_FALSE=100, - POP_JUMP_IF_NONE=101, - POP_JUMP_IF_NOT_NONE=102, - POP_JUMP_IF_TRUE=103, - RAISE_VARARGS=104, - RERAISE=105, - SEND=106, - SET_ADD=107, - SET_FUNCTION_ATTRIBUTE=108, - SET_UPDATE=109, - STORE_ATTR=110, - STORE_DEREF=111, - STORE_FAST=112, - STORE_FAST_LOAD_FAST=113, - STORE_FAST_STORE_FAST=114, - STORE_GLOBAL=115, - STORE_NAME=116, - SWAP=117, - UNPACK_EX=118, - UNPACK_SEQUENCE=119, - YIELD_VALUE=120, + GET_LEN=16, + INTERPRETER_EXIT=18, + LOAD_BUILD_CLASS=19, + LOAD_LOCALS=20, + MAKE_FUNCTION=21, + MATCH_KEYS=22, + MATCH_MAPPING=23, + MATCH_SEQUENCE=24, + NOP=25, + NOT_TAKEN=26, + POP_EXCEPT=27, + POP_ITER=28, + POP_TOP=29, + PUSH_EXC_INFO=30, + PUSH_NULL=31, + RETURN_GENERATOR=32, + RETURN_VALUE=33, + SETUP_ANNOTATIONS=34, + STORE_SLICE=35, + STORE_SUBSCR=36, + TO_BOOL=37, + UNARY_INVERT=38, + UNARY_NEGATIVE=39, + UNARY_NOT=40, + WITH_EXCEPT_START=41, + BINARY_OP=42, + BUILD_INTERPOLATION=43, + BUILD_LIST=44, + BUILD_MAP=45, + BUILD_SET=46, + BUILD_SLICE=47, + BUILD_STRING=48, + BUILD_TUPLE=49, + CALL=50, + CALL_INTRINSIC_1=51, + CALL_INTRINSIC_2=52, + CALL_KW=53, + COMPARE_OP=54, + CONTAINS_OP=55, + CONVERT_VALUE=56, + COPY=57, + COPY_FREE_VARS=58, + DELETE_ATTR=59, + DELETE_DEREF=60, + DELETE_FAST=61, + DELETE_GLOBAL=62, + DELETE_NAME=63, + DICT_MERGE=64, + DICT_UPDATE=65, + END_ASYNC_FOR=66, + EXTENDED_ARG=67, + FOR_ITER=68, + GET_AWAITABLE=69, + GET_ITER=70, + IMPORT_FROM=71, + IMPORT_NAME=72, + IS_OP=73, + JUMP_BACKWARD=74, + JUMP_BACKWARD_NO_INTERRUPT=75, + JUMP_FORWARD=76, + LIST_APPEND=77, + LIST_EXTEND=78, + LOAD_ATTR=79, + LOAD_COMMON_CONSTANT=80, + LOAD_CONST=81, + LOAD_DEREF=82, + LOAD_FAST=83, + LOAD_FAST_AND_CLEAR=84, + LOAD_FAST_BORROW=85, + LOAD_FAST_BORROW_LOAD_FAST_BORROW=86, + LOAD_FAST_CHECK=87, + LOAD_FAST_LOAD_FAST=88, + LOAD_FROM_DICT_OR_DEREF=89, + LOAD_FROM_DICT_OR_GLOBALS=90, + LOAD_GLOBAL=91, + LOAD_NAME=92, + LOAD_SMALL_INT=93, + LOAD_SPECIAL=94, + LOAD_SUPER_ATTR=95, + MAKE_CELL=96, + MAP_ADD=97, + MATCH_CLASS=98, + POP_JUMP_IF_FALSE=99, + POP_JUMP_IF_NONE=100, + POP_JUMP_IF_NOT_NONE=101, + POP_JUMP_IF_TRUE=102, + RAISE_VARARGS=103, + RERAISE=104, + SEND=105, + SET_ADD=106, + SET_FUNCTION_ATTRIBUTE=107, + SET_UPDATE=108, + STORE_ATTR=109, + STORE_DEREF=110, + STORE_FAST=111, + STORE_FAST_LOAD_FAST=112, + STORE_FAST_STORE_FAST=113, + STORE_GLOBAL=114, + STORE_NAME=115, + SWAP=116, + UNPACK_EX=117, + UNPACK_SEQUENCE=118, + YIELD_VALUE=119, INSTRUMENTED_END_FOR=233, INSTRUMENTED_POP_ITER=234, INSTRUMENTED_END_SEND=235, @@ -372,5 +371,5 @@ STORE_FAST_MAYBE_NULL=266, ) -HAVE_ARGUMENT = 43 +HAVE_ARGUMENT = 41 MIN_INSTRUMENTED_OPCODE = 233 diff --git a/Lib/_py_warnings.py b/Lib/_py_warnings.py index d5a9cec86f3674..81a386c4487d95 100644 --- a/Lib/_py_warnings.py +++ b/Lib/_py_warnings.py @@ -703,8 +703,8 @@ def __enter__(self): context = None self._filters = self._module.filters self._module.filters = self._filters[:] - self._showwarning = self._module.showwarning self._showwarnmsg_impl = self._module._showwarnmsg_impl + self._showwarning = self._module.showwarning self._module._filters_mutated_lock_held() if self._record: if _use_context: @@ -712,9 +712,9 @@ def __enter__(self): else: log = [] self._module._showwarnmsg_impl = log.append - # Reset showwarning() to the default implementation to make sure - # that _showwarnmsg() calls _showwarnmsg_impl() - self._module.showwarning = self._module._showwarning_orig + # Reset showwarning() to the default implementation to make sure + # that _showwarnmsg() calls _showwarnmsg_impl() + self._module.showwarning = self._module._showwarning_orig else: log = None if self._filter is not None: @@ -729,8 +729,8 @@ def __exit__(self, *exc_info): self._module._warnings_context.set(self._saved_context) else: self._module.filters = self._filters - self._module.showwarning = self._showwarning self._module._showwarnmsg_impl = self._showwarnmsg_impl + self._module.showwarning = self._showwarning self._module._filters_mutated_lock_held() diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py index 2098d0a54aba31..a22b0297b24ea0 100644 --- a/Lib/_pyrepl/_module_completer.py +++ b/Lib/_pyrepl/_module_completer.py @@ -3,6 +3,7 @@ import importlib import os import pkgutil +import re import sys import token import tokenize @@ -16,17 +17,31 @@ TYPE_CHECKING = False if TYPE_CHECKING: + from types import ModuleType from typing import Any, Iterable, Iterator, Mapping + from .types import CompletionAction HARDCODED_SUBMODULES = { # Standard library submodules that are not detected by pkgutil.iter_modules # but can be imported, so should be proposed in completion "collections": ["abc"], + "math": ["integer"], "os": ["path"], "xml.parsers.expat": ["errors", "model"], } +AUTO_IMPORT_DENYLIST = { + # Standard library modules/submodules that have import side effects + # and must not be automatically imported to complete attributes + re.compile(r"antigravity"), # Calls webbrowser.open + re.compile(r"idlelib\..+"), # May open IDLE GUI + re.compile(r"test\..+"), # Various side-effects + re.compile(r"this"), # Prints to stdout + re.compile(r"_ios_support"), # Spawns a subprocess + re.compile(r".+\.__main__"), # Should not be imported +} + def make_default_module_completer() -> ModuleCompleter: # Inside pyrepl, __package__ is set to None by default @@ -52,11 +67,17 @@ class ModuleCompleter: def __init__(self, namespace: Mapping[str, Any] | None = None) -> None: self.namespace = namespace or {} self._global_cache: list[pkgutil.ModuleInfo] = [] + self._failed_imports: set[str] = set() self._curr_sys_path: list[str] = sys.path[:] self._stdlib_path = os.path.dirname(importlib.__path__[0]) - def get_completions(self, line: str) -> list[str] | None: - """Return the next possible import completions for 'line'.""" + def get_completions(self, line: str) -> tuple[list[str], CompletionAction | None] | None: + """Return the next possible import completions for 'line'. + + For attributes completion, if the module to complete from is not + imported, also return an action (prompt + callback to run if the + user press TAB again) to import the module. + """ result = ImportParser(line).parse() if not result: return None @@ -65,24 +86,26 @@ def get_completions(self, line: str) -> list[str] | None: except Exception: # Some unexpected error occurred, make it look like # no completions are available - return [] + return [], None - def complete(self, from_name: str | None, name: str | None) -> list[str]: + def complete(self, from_name: str | None, name: str | None) -> tuple[list[str], CompletionAction | None]: if from_name is None: # import x.y.z assert name is not None path, prefix = self.get_path_and_prefix(name) modules = self.find_modules(path, prefix) - return [self.format_completion(path, module) for module in modules] + return [self.format_completion(path, module) for module in modules], None if name is None: # from x.y.z path, prefix = self.get_path_and_prefix(from_name) modules = self.find_modules(path, prefix) - return [self.format_completion(path, module) for module in modules] + return [self.format_completion(path, module) for module in modules], None # from x.y import z - return self.find_modules(from_name, name) + submodules = self.find_modules(from_name, name) + attributes, action = self.find_attributes(from_name, name) + return sorted({*submodules, *attributes}), action def find_modules(self, path: str, prefix: str) -> list[str]: """Find all modules under 'path' that start with 'prefix'.""" @@ -100,23 +123,25 @@ def _find_modules(self, path: str, prefix: str) -> list[str]: if self.is_suggestion_match(module.name, prefix)] return sorted(builtin_modules + third_party_modules) - if path.startswith('.'): - # Convert relative path to absolute path - package = self.namespace.get('__package__', '') - path = self.resolve_relative_name(path, package) # type: ignore[assignment] - if path is None: - return [] + path = self._resolve_relative_path(path) # type: ignore[assignment] + if path is None: + return [] modules: Iterable[pkgutil.ModuleInfo] = self.global_cache imported_module = sys.modules.get(path.split('.')[0]) if imported_module: - # Filter modules to those who name and specs match the + # Filter modules to those whose name and specs match the # imported module to avoid invalid suggestions spec = imported_module.__spec__ if spec: + def _safe_find_spec(mod: pkgutil.ModuleInfo) -> bool: + try: + return mod.module_finder.find_spec(mod.name, None) == spec + except Exception: + return False modules = [mod for mod in modules if mod.name == spec.name - and mod.module_finder.find_spec(mod.name, None) == spec] + and _safe_find_spec(mod)] else: modules = [] @@ -141,6 +166,32 @@ def _is_stdlib_module(self, module_info: pkgutil.ModuleInfo) -> bool: return (isinstance(module_info.module_finder, FileFinder) and module_info.module_finder.path == self._stdlib_path) + def find_attributes(self, path: str, prefix: str) -> tuple[list[str], CompletionAction | None]: + """Find all attributes of module 'path' that start with 'prefix'.""" + attributes, action = self._find_attributes(path, prefix) + # Filter out invalid attribute names + # (for example those containing dashes that cannot be imported with 'import') + return [attr for attr in attributes if attr.isidentifier()], action + + def _find_attributes(self, path: str, prefix: str) -> tuple[list[str], CompletionAction | None]: + path = self._resolve_relative_path(path) # type: ignore[assignment] + if path is None: + return [], None + + imported_module = sys.modules.get(path) + if not imported_module: + if path in self._failed_imports: # Do not propose to import again + return [], None + imported_module = self._maybe_import_module(path) + if not imported_module: + return [], self._get_import_completion_action(path) + try: + module_attributes = dir(imported_module) + except Exception: + module_attributes = [] + return [attr_name for attr_name in module_attributes + if self.is_suggestion_match(attr_name, prefix)], None + def is_suggestion_match(self, module_name: str, prefix: str) -> bool: if prefix: return module_name.startswith(prefix) @@ -185,6 +236,13 @@ def format_completion(self, path: str, module: str) -> str: return f'{path}{module}' return f'{path}.{module}' + def _resolve_relative_path(self, path: str) -> str | None: + """Resolve a relative import path to absolute. Returns None if unresolvable.""" + if path.startswith('.'): + package = self.namespace.get('__package__', '') + return self.resolve_relative_name(path, package) + return path + def resolve_relative_name(self, name: str, package: str) -> str | None: """Resolve a relative module name to an absolute name. @@ -209,8 +267,39 @@ def global_cache(self) -> list[pkgutil.ModuleInfo]: if not self._global_cache or self._curr_sys_path != sys.path: self._curr_sys_path = sys.path[:] self._global_cache = list(pkgutil.iter_modules()) + self._failed_imports.clear() # retry on sys.path change return self._global_cache + def _maybe_import_module(self, fqname: str) -> ModuleType | None: + if any(pattern.fullmatch(fqname) for pattern in AUTO_IMPORT_DENYLIST): + # Special-cased modules with known import side-effects + return None + root = fqname.split(".")[0] + mod_info = next((m for m in self.global_cache if m.name == root), None) + if not mod_info or not self._is_stdlib_module(mod_info): + # Only import stdlib modules (no risk of import side-effects) + return None + try: + return importlib.import_module(fqname) + except Exception: + sys.modules.pop(fqname, None) # Clean half-imported module + return None + + def _get_import_completion_action(self, path: str) -> CompletionAction: + prompt = ("[ module not imported, press again to import it " + "and propose attributes ]") + + def _do_import() -> str | None: + try: + importlib.import_module(path) + return None + except Exception as exc: + sys.modules.pop(path, None) # Clean half-imported module + self._failed_imports.add(path) + return f"[ error during import: {exc} ]" + + return (prompt, _do_import) + class ImportParser: """ diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py index 9d2d43be5144e8..39d0a8af5dfaea 100644 --- a/Lib/_pyrepl/completing_reader.py +++ b/Lib/_pyrepl/completing_reader.py @@ -29,8 +29,9 @@ # types Command = commands.Command -if False: - from .types import KeySpec, CommandName +TYPE_CHECKING = False +if TYPE_CHECKING: + from .types import KeySpec, CommandName, CompletionAction def prefix(wordlist: list[str], j: int = 0) -> str: @@ -168,22 +169,34 @@ def do(self) -> None: r: CompletingReader r = self.reader # type: ignore[assignment] last_is_completer = r.last_command_is(self.__class__) + if r.cmpltn_action: + if last_is_completer: # double-tab: execute action + msg = r.cmpltn_action[1]() + r.cmpltn_action = None # consumed + if msg: + r.msg = msg + else: # other input since last tab: cancel action + r.cmpltn_action = None + immutable_completions = r.assume_immutable_completions completions_unchangable = last_is_completer and immutable_completions stem = r.get_stem() if not completions_unchangable: - r.cmpltn_menu_choices = r.get_completions(stem) + r.cmpltn_menu_choices, r.cmpltn_action = r.get_completions(stem) completions = r.cmpltn_menu_choices if not completions: - r.error("no matches") + if not r.cmpltn_action: + r.error("no matches") elif len(completions) == 1: - if completions_unchangable and len(completions[0]) == len(stem): + completion = stripcolor(completions[0]) + if completions_unchangable and len(completion) == len(stem): r.msg = "[ sole completion ]" r.dirty = True - r.insert(completions[0][len(stem):]) + r.insert(completion[len(stem):]) else: - p = prefix(completions, len(stem)) + clean_completions = [stripcolor(word) for word in completions] + p = prefix(clean_completions, len(stem)) if p: r.insert(p) if last_is_completer: @@ -195,13 +208,23 @@ def do(self) -> None: r.dirty = True elif not r.cmpltn_menu_visible: r.cmpltn_message_visible = True - if stem + p in completions: + if stem + p in clean_completions: r.msg = "[ complete but not unique ]" r.dirty = True else: r.msg = "[ not unique ]" r.dirty = True + if r.cmpltn_action: + if r.msg and r.cmpltn_message_visible: + # There is already a message (eg. [ not unique ]) that + # would conflict for next tab: cancel action + r.cmpltn_action = None + else: + r.msg = r.cmpltn_action[0] + r.cmpltn_message_visible = True + r.dirty = True + class self_insert(commands.self_insert): def do(self) -> None: @@ -215,7 +238,7 @@ def do(self) -> None: r.cmpltn_reset() else: completions = [w for w in r.cmpltn_menu_choices - if w.startswith(stem)] + if stripcolor(w).startswith(stem)] if completions: r.cmpltn_menu, r.cmpltn_menu_end = build_menu( r.console, completions, 0, @@ -240,6 +263,7 @@ class CompletingReader(Reader): cmpltn_message_visible: bool = field(init=False) cmpltn_menu_end: int = field(init=False) cmpltn_menu_choices: list[str] = field(init=False) + cmpltn_action: CompletionAction | None = field(init=False) def __post_init__(self) -> None: super().__post_init__() @@ -281,6 +305,7 @@ def cmpltn_reset(self) -> None: self.cmpltn_message_visible = False self.cmpltn_menu_end = 0 self.cmpltn_menu_choices = [] + self.cmpltn_action = None def get_stem(self) -> str: st = self.syntax_table @@ -291,8 +316,8 @@ def get_stem(self) -> str: p -= 1 return ''.join(b[p+1:self.pos]) - def get_completions(self, stem: str) -> list[str]: - return [] + def get_completions(self, stem: str) -> tuple[list[str], CompletionAction | None]: + return [], None def get_line(self) -> str: """Return the current line until the cursor position.""" diff --git a/Lib/_pyrepl/fancycompleter.py b/Lib/_pyrepl/fancycompleter.py new file mode 100644 index 00000000000000..5b5b7ae5f2bb59 --- /dev/null +++ b/Lib/_pyrepl/fancycompleter.py @@ -0,0 +1,210 @@ +# Copyright 2010-2025 Antonio Cuni +# Daniel Hahler +# +# All Rights Reserved +"""Colorful tab completion for Python prompt""" +from _colorize import ANSIColors, get_colors, get_theme +import rlcompleter +import keyword +import types + +class Completer(rlcompleter.Completer): + """ + When doing something like a.b., keep the full a.b.attr completion + stem so readline-style completion can keep refining the menu as you type. + + Optionally, display the various completions in different colors + depending on the type. + """ + def __init__( + self, + namespace=None, + *, + use_colors='auto', + consider_getitems=True, + ): + from _pyrepl import readline + rlcompleter.Completer.__init__(self, namespace) + if use_colors == 'auto': + # use colors only if we can + use_colors = get_colors().RED != "" + self.use_colors = use_colors + self.consider_getitems = consider_getitems + + if self.use_colors: + # In GNU readline, this prevents escaping of ANSI control + # characters in completion results. pyrepl's parse_and_bind() + # is a no-op, but pyrepl handles ANSI sequences natively + # via real_len()/stripcolor(). + readline.parse_and_bind('set dont-escape-ctrl-chars on') + self.theme = get_theme() + else: + self.theme = None + + if self.consider_getitems: + delims = readline.get_completer_delims() + delims = delims.replace('[', '') + delims = delims.replace(']', '') + readline.set_completer_delims(delims) + + def complete(self, text, state): + # if you press at the beginning of a line, insert an actual + # \t. Else, trigger completion. + if text == "": + return ('\t', None)[state] + else: + return rlcompleter.Completer.complete(self, text, state) + + def _callable_postfix(self, val, word): + # disable automatic insertion of '(' for global callables + return word + + def _callable_attr_postfix(self, val, word): + return rlcompleter.Completer._callable_postfix(self, val, word) + + def global_matches(self, text): + names = rlcompleter.Completer.global_matches(self, text) + prefix = commonprefix(names) + if prefix and prefix != text: + return [prefix] + + names.sort() + values = [] + for name in names: + clean_name = name.rstrip(': ') + if keyword.iskeyword(clean_name) or keyword.issoftkeyword(clean_name): + values.append(None) + else: + try: + values.append(eval(name, self.namespace)) + except Exception: + values.append(None) + if self.use_colors and names: + return self.colorize_matches(names, values) + return names + + def attr_matches(self, text): + try: + expr, attr, names, values = self._attr_matches(text) + except ValueError: + return [] + + if not names: + return [] + + if len(names) == 1: + # No coloring: when returning a single completion, readline + # inserts it directly into the prompt, so ANSI codes would + # appear as literal characters. + return [self._callable_attr_postfix(values[0], f'{expr}.{names[0]}')] + + prefix = commonprefix(names) + if prefix and prefix != attr: + return [f'{expr}.{prefix}'] # autocomplete prefix + + names = [f'{expr}.{name}' for name in names] + if self.use_colors: + return self.colorize_matches(names, values) + + if prefix: + names.append(' ') + return names + + def _attr_matches(self, text): + expr, attr = text.rsplit('.', 1) + if '(' in expr or ')' in expr: # don't call functions + return expr, attr, [], [] + try: + thisobject = eval(expr, self.namespace) + except Exception: + return expr, attr, [], [] + + # get the content of the object, except __builtins__ + words = set(dir(thisobject)) - {'__builtins__'} + + if hasattr(thisobject, '__class__'): + words.add('__class__') + words.update(rlcompleter.get_class_members(thisobject.__class__)) + names = [] + values = [] + n = len(attr) + if attr == '': + noprefix = '_' + elif attr == '_': + noprefix = '__' + else: + noprefix = None + + # sort the words now to make sure to return completions in + # alphabetical order. It's easier to do it now, else we would need to + # sort 'names' later but make sure that 'values' in kept in sync, + # which is annoying. + words = sorted(words) + while True: + for word in words: + if ( + word[:n] == attr + and not (noprefix and word[:n+1] == noprefix) + ): + # Mirror rlcompleter's safeguards so completion does not + # call properties or reify lazy module attributes. + if isinstance(getattr(type(thisobject), word, None), property): + value = None + elif ( + isinstance(thisobject, types.ModuleType) + and isinstance( + thisobject.__dict__.get(word), + types.LazyImportType, + ) + ): + value = thisobject.__dict__.get(word) + else: + value = getattr(thisobject, word, None) + + names.append(word) + values.append(value) + if names or not noprefix: + break + if noprefix == '_': + noprefix = '__' + else: + noprefix = None + + return expr, attr, names, values + + def colorize_matches(self, names, values): + matches = [self._color_for_obj(i, name, obj) + for i, (name, obj) + in enumerate(zip(names, values))] + # We add a space at the end to prevent the automatic completion of the + # common prefix, which is the ANSI escape sequence. + matches.append(' ') + return matches + + def _color_for_obj(self, i, name, value): + t = type(value) + color = self._color_by_type(t) + # Encode the match index into a fake escape sequence that + # stripcolor() can still remove once i reaches four digits. + N = f"\x1b[{i // 100:03d};{i % 100:02d}m" + return f"{N}{color}{name}{ANSIColors.RESET}" + + def _color_by_type(self, t): + typename = t.__name__ + # this is needed e.g. to turn method-wrapper into method_wrapper, + # because if we want _colorize.FancyCompleter to be "dataclassable" + # our keys need to be valid identifiers. + typename = typename.replace('-', '_').replace('.', '_') + return getattr(self.theme.fancycompleter, typename, ANSIColors.RESET) + + +def commonprefix(names): + """Return the common prefix of all 'names'""" + if not names: + return '' + s1 = min(names) + s2 = max(names) + for i, c in enumerate(s1): + if c != s2[i]: + return s1[:i] + return s1 diff --git a/Lib/_pyrepl/pager.py b/Lib/_pyrepl/pager.py index 1fddc63e3ee3ad..92afaa5933a225 100644 --- a/Lib/_pyrepl/pager.py +++ b/Lib/_pyrepl/pager.py @@ -138,7 +138,7 @@ def pipe_pager(text: str, cmd: str, title: str = '') -> None: '.' '?e (END):?pB %pB\\%..' ' (press h for help or q to quit)') - env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string) + env['LESS'] = '-RcmPm{0}$PM{0}$'.format(prompt_string) proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, errors='backslashreplace', env=env) assert proc.stdin is not None diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 9ab92f64d1ef63..f35a99fb06a3f9 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -381,9 +381,17 @@ def calc_screen(self) -> list[str]: self.screeninfo = screeninfo self.cxy = self.pos2xy() if self.msg: + width = self.console.width for mline in self.msg.split("\n"): - screen.append(mline) - screeninfo.append((0, [])) + # If self.msg is larger than console width, make it fit + # TODO: try to split between words? + if not mline: + screen.append("") + screeninfo.append((0, [])) + continue + for r in range((len(mline) - 1) // width + 1): + screen.append(mline[r * width : (r + 1) * width]) + screeninfo.append((0, [])) self.last_refresh_cache.update_cache(self, screen, screeninfo) return screen @@ -628,7 +636,6 @@ def suspend_colorization(self) -> SimpleContextManager: finally: self.can_colorize = old_can_colorize - def finish(self) -> None: """Called when a command signals that we're finished.""" pass diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 23b8fa6b9c7625..687084601e77c1 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -40,6 +40,7 @@ from .completing_reader import CompletingReader from .console import Console as ConsoleType from ._module_completer import ModuleCompleter, make_default_module_completer +from .fancycompleter import Completer as FancyCompleter Console: type[ConsoleType] _error: tuple[type[Exception], ...] | type[Exception] @@ -55,7 +56,7 @@ # types Command = commands.Command from collections.abc import Callable, Collection -from .types import Callback, Completer, KeySpec, CommandName +from .types import Callback, Completer, KeySpec, CommandName, CompletionAction TYPE_CHECKING = False @@ -134,7 +135,7 @@ def get_stem(self) -> str: p -= 1 return "".join(b[p + 1 : self.pos]) - def get_completions(self, stem: str) -> list[str]: + def get_completions(self, stem: str) -> tuple[list[str], CompletionAction | None]: module_completions = self.get_module_completions() if module_completions is not None: return module_completions @@ -144,7 +145,7 @@ def get_completions(self, stem: str) -> list[str]: while p > 0 and b[p - 1] != "\n": p -= 1 num_spaces = 4 - ((self.pos - p) % 4) - return [" " * num_spaces] + return [" " * num_spaces], None result = [] function = self.config.readline_completer if function is not None: @@ -165,9 +166,9 @@ def get_completions(self, stem: str) -> list[str]: # emulate the behavior of the standard readline that sorts # the completions before displaying them. result.sort() - return result + return result, None - def get_module_completions(self) -> list[str] | None: + def get_module_completions(self) -> tuple[list[str], CompletionAction | None] | None: line = self.get_line() return self.config.module_completer.get_completions(line) @@ -609,7 +610,12 @@ def _setup(namespace: Mapping[str, Any]) -> None: if not isinstance(namespace, dict): namespace = dict(namespace) _wrapper.config.module_completer = ModuleCompleter(namespace) - _wrapper.config.readline_completer = RLCompleter(namespace).complete + use_basic_completer = ( + not sys.flags.ignore_environment + and os.getenv("PYTHON_BASIC_COMPLETER") + ) + completer_cls = RLCompleter if use_basic_completer else FancyCompleter + _wrapper.config.readline_completer = completer_cls(namespace).complete # this is not really what readline.c does. Better than nothing I guess import builtins diff --git a/Lib/_pyrepl/types.py b/Lib/_pyrepl/types.py index c5b7ebc1a406bd..e19607bf18e8b1 100644 --- a/Lib/_pyrepl/types.py +++ b/Lib/_pyrepl/types.py @@ -8,3 +8,4 @@ type Completer = Callable[[str, int], str | None] type CharBuffer = list[str] type CharWidths = list[int] +type CompletionAction = tuple[str, Callable[[], str | None]] diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index 25d7ac1bd0b14e..7175d57a9e319e 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -19,9 +19,9 @@ ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]") ZERO_WIDTH_BRACKET = re.compile(r"\x01.*?\x02") ZERO_WIDTH_TRANS = str.maketrans({"\x01": "", "\x02": ""}) -IDENTIFIERS_AFTER = {"def", "class"} -KEYWORD_CONSTANTS = {"True", "False", "None"} -BUILTINS = {str(name) for name in dir(builtins) if not name.startswith('_')} +IDENTIFIERS_AFTER = frozenset({"def", "class"}) +KEYWORD_CONSTANTS = frozenset({"True", "False", "None"}) +BUILTINS = frozenset({str(name) for name in dir(builtins) if not name.startswith('_')}) def THEME(**kwargs): @@ -226,8 +226,8 @@ def gen_colors_from_token_stream( yield ColorSpan(span, "builtin") -keyword_first_sets_match = {"False", "None", "True", "await", "lambda", "not"} -keyword_first_sets_case = {"False", "None", "True"} +keyword_first_sets_match = frozenset({"False", "None", "True", "await", "lambda", "not"}) +keyword_first_sets_case = frozenset({"False", "None", "True"}) def is_soft_keyword_used(*tokens: TI | None) -> bool: diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index df8fb5e4c62079..9fee2564114339 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -1037,13 +1037,26 @@ def get_annotations( obj_globals = obj_locals = unwrap = None if unwrap is not None: + # Use an id-based visited set to detect cycles in the __wrapped__ + # and functools.partial.func chain (e.g. f.__wrapped__ = f). + # On cycle detection we stop and use whatever __globals__ we have + # found so far, mirroring the approach of inspect.unwrap(). + _seen_ids = {id(unwrap)} while True: if hasattr(unwrap, "__wrapped__"): - unwrap = unwrap.__wrapped__ + candidate = unwrap.__wrapped__ + if id(candidate) in _seen_ids: + break + _seen_ids.add(id(candidate)) + unwrap = candidate continue if functools := sys.modules.get("functools"): if isinstance(unwrap, functools.partial): - unwrap = unwrap.func + candidate = unwrap.func + if id(candidate) in _seen_ids: + break + _seen_ids.add(id(candidate)) + unwrap = candidate continue break if hasattr(unwrap, "__globals__"): diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 77c70aaa7b986e..7a6837546d930f 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -14,6 +14,7 @@ """ import collections +import contextvars import collections.abc import concurrent.futures import errno @@ -290,6 +291,7 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, self._ssl_shutdown_timeout = ssl_shutdown_timeout self._serving = False self._serving_forever_fut = None + self._context = contextvars.copy_context() def __repr__(self): return f'<{self.__class__.__name__} sockets={self.sockets!r}>' @@ -319,7 +321,7 @@ def _start_serving(self): self._loop._start_serving( self._protocol_factory, sock, self._ssl_context, self, self._backlog, self._ssl_handshake_timeout, - self._ssl_shutdown_timeout) + self._ssl_shutdown_timeout, context=self._context) def get_loop(self): return self._loop @@ -509,7 +511,8 @@ def _make_ssl_transport( extra=None, server=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, - call_connection_made=True): + call_connection_made=True, + context=None): """Create SSL transport.""" raise NotImplementedError @@ -1213,9 +1216,10 @@ async def _create_connection_transport( self, sock, protocol_factory, ssl, server_hostname, server_side=False, ssl_handshake_timeout=None, - ssl_shutdown_timeout=None): + ssl_shutdown_timeout=None, context=None): sock.setblocking(False) + context = context if context is not None else contextvars.copy_context() protocol = protocol_factory() waiter = self.create_future() @@ -1225,9 +1229,10 @@ async def _create_connection_transport( sock, protocol, sslcontext, waiter, server_side=server_side, server_hostname=server_hostname, ssl_handshake_timeout=ssl_handshake_timeout, - ssl_shutdown_timeout=ssl_shutdown_timeout) + ssl_shutdown_timeout=ssl_shutdown_timeout, + context=context) else: - transport = self._make_socket_transport(sock, protocol, waiter) + transport = self._make_socket_transport(sock, protocol, waiter, context=context) try: await waiter diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 3fa93b14a6787f..2dc1569d780791 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -642,7 +642,7 @@ def __init__(self, proactor): signal.set_wakeup_fd(self._csock.fileno()) def _make_socket_transport(self, sock, protocol, waiter=None, - extra=None, server=None): + extra=None, server=None, context=None): return _ProactorSocketTransport(self, sock, protocol, waiter, extra, server) @@ -651,7 +651,7 @@ def _make_ssl_transport( *, server_side=False, server_hostname=None, extra=None, server=None, ssl_handshake_timeout=None, - ssl_shutdown_timeout=None): + ssl_shutdown_timeout=None, context=None): ssl_protocol = sslproto.SSLProtocol( self, protocol, sslcontext, waiter, server_side, server_hostname, @@ -837,7 +837,7 @@ def _write_to_self(self): def _start_serving(self, protocol_factory, sock, sslcontext=None, server=None, backlog=100, ssl_handshake_timeout=None, - ssl_shutdown_timeout=None): + ssl_shutdown_timeout=None, context=None): def loop(f=None): try: diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index 084fccaaff2ff7..756216fac80932 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -37,7 +37,7 @@ class Queue(mixins._LoopBoundMixin): is an integer greater than 0, then "await put()" will block when the queue reaches maxsize, until an item is removed by get(). - Unlike the standard library Queue, you can reliably know this Queue's size + Unlike queue.Queue, you can reliably know this Queue's size with qsize(), since your single-threaded asyncio application won't be interrupted between calling qsize() and doing an operation on the Queue. """ diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index ff7e16df3c6273..961dbfb4b96303 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -67,10 +67,10 @@ def __init__(self, selector=None): self._transports = weakref.WeakValueDictionary() def _make_socket_transport(self, sock, protocol, waiter=None, *, - extra=None, server=None): + extra=None, server=None, context=None): self._ensure_fd_no_transport(sock) return _SelectorSocketTransport(self, sock, protocol, waiter, - extra, server) + extra, server, context=context) def _make_ssl_transport( self, rawsock, protocol, sslcontext, waiter=None, @@ -78,16 +78,17 @@ def _make_ssl_transport( extra=None, server=None, ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, + context=None, ): self._ensure_fd_no_transport(rawsock) ssl_protocol = sslproto.SSLProtocol( self, protocol, sslcontext, waiter, server_side, server_hostname, ssl_handshake_timeout=ssl_handshake_timeout, - ssl_shutdown_timeout=ssl_shutdown_timeout + ssl_shutdown_timeout=ssl_shutdown_timeout, ) _SelectorSocketTransport(self, rawsock, ssl_protocol, - extra=extra, server=server) + extra=extra, server=server, context=context) return ssl_protocol._app_transport def _make_datagram_transport(self, sock, protocol, @@ -159,16 +160,16 @@ def _write_to_self(self): def _start_serving(self, protocol_factory, sock, sslcontext=None, server=None, backlog=100, ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, - ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT): + ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, context=None): self._add_reader(sock.fileno(), self._accept_connection, protocol_factory, sock, sslcontext, server, backlog, - ssl_handshake_timeout, ssl_shutdown_timeout) + ssl_handshake_timeout, ssl_shutdown_timeout, context) def _accept_connection( self, protocol_factory, sock, sslcontext=None, server=None, backlog=100, ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, - ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT): + ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, context=None): # This method is only called once for each event loop tick where the # listening socket has triggered an EVENT_READ. There may be multiple # connections waiting for an .accept() so it is called in a loop. @@ -204,21 +205,22 @@ def _accept_connection( self._start_serving, protocol_factory, sock, sslcontext, server, backlog, ssl_handshake_timeout, - ssl_shutdown_timeout) + ssl_shutdown_timeout, context) else: raise # The event loop will catch, log and ignore it. else: extra = {'peername': addr} + conn_context = context.copy() if context is not None else None accept = self._accept_connection2( protocol_factory, conn, extra, sslcontext, server, - ssl_handshake_timeout, ssl_shutdown_timeout) - self.create_task(accept) + ssl_handshake_timeout, ssl_shutdown_timeout, context=conn_context) + self.create_task(accept, context=conn_context) async def _accept_connection2( self, protocol_factory, conn, extra, sslcontext=None, server=None, ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, - ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT): + ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, context=None): protocol = None transport = None try: @@ -229,11 +231,12 @@ async def _accept_connection2( conn, protocol, sslcontext, waiter=waiter, server_side=True, extra=extra, server=server, ssl_handshake_timeout=ssl_handshake_timeout, - ssl_shutdown_timeout=ssl_shutdown_timeout) + ssl_shutdown_timeout=ssl_shutdown_timeout, + context=context) else: transport = self._make_socket_transport( conn, protocol, waiter=waiter, extra=extra, - server=server) + server=server, context=context) try: await waiter @@ -275,9 +278,9 @@ def _ensure_fd_no_transport(self, fd): f'File descriptor {fd!r} is used by transport ' f'{transport!r}') - def _add_reader(self, fd, callback, *args): + def _add_reader(self, fd, callback, *args, context=None): self._check_closed() - handle = events.Handle(callback, args, self, None) + handle = events.Handle(callback, args, self, context=context) key = self._selector.get_map().get(fd) if key is None: self._selector.register(fd, selectors.EVENT_READ, @@ -309,9 +312,9 @@ def _remove_reader(self, fd): else: return False - def _add_writer(self, fd, callback, *args): + def _add_writer(self, fd, callback, *args, context=None): self._check_closed() - handle = events.Handle(callback, args, self, None) + handle = events.Handle(callback, args, self, context=context) key = self._selector.get_map().get(fd) if key is None: self._selector.register(fd, selectors.EVENT_WRITE, @@ -770,7 +773,7 @@ class _SelectorTransport(transports._FlowControlMixin, # exception) _sock = None - def __init__(self, loop, sock, protocol, extra=None, server=None): + def __init__(self, loop, sock, protocol, extra=None, server=None, context=None): super().__init__(extra, loop) self._extra['socket'] = trsock.TransportSocket(sock) try: @@ -784,12 +787,13 @@ def __init__(self, loop, sock, protocol, extra=None, server=None): self._extra['peername'] = None self._sock = sock self._sock_fd = sock.fileno() - + self._context = context self._protocol_connected = False self.set_protocol(protocol) self._server = server self._buffer = collections.deque() + self._buffer_size = 0 self._conn_lost = 0 # Set when call to connection_lost scheduled. self._closing = False # Set when close() called. self._paused = False # Set when pause_reading() called @@ -866,7 +870,7 @@ def close(self): if not self._buffer: self._conn_lost += 1 self._loop._remove_writer(self._sock_fd) - self._loop.call_soon(self._call_connection_lost, None) + self._call_soon(self._call_connection_lost, None) def __del__(self, _warn=warnings.warn): if self._sock is not None: @@ -894,12 +898,13 @@ def _force_close(self, exc): return if self._buffer: self._buffer.clear() + self._buffer_size = 0 self._loop._remove_writer(self._sock_fd) if not self._closing: self._closing = True self._loop._remove_reader(self._sock_fd) self._conn_lost += 1 - self._loop.call_soon(self._call_connection_lost, exc) + self._call_soon(self._call_connection_lost, exc) def _call_connection_lost(self, exc): try: @@ -916,13 +921,18 @@ def _call_connection_lost(self, exc): self._server = None def get_write_buffer_size(self): - return sum(map(len, self._buffer)) + return self._buffer_size def _add_reader(self, fd, callback, *args): if not self.is_reading(): return - self._loop._add_reader(fd, callback, *args) + self._loop._add_reader(fd, callback, *args, context=self._context) + def _add_writer(self, fd, callback, *args): + self._loop._add_writer(fd, callback, *args, context=self._context) + + def _call_soon(self, callback, *args): + self._loop.call_soon(callback, *args, context=self._context) class _SelectorSocketTransport(_SelectorTransport): @@ -930,10 +940,9 @@ class _SelectorSocketTransport(_SelectorTransport): _sendfile_compatible = constants._SendfileMode.TRY_NATIVE def __init__(self, loop, sock, protocol, waiter=None, - extra=None, server=None): - + extra=None, server=None, context=None): self._read_ready_cb = None - super().__init__(loop, sock, protocol, extra, server) + super().__init__(loop, sock, protocol, extra, server, context) self._eof = False self._empty_waiter = None if _HAS_SENDMSG: @@ -945,14 +954,12 @@ def __init__(self, loop, sock, protocol, waiter=None, # decreases the latency (in some cases significantly.) base_events._set_nodelay(self._sock) - self._loop.call_soon(self._protocol.connection_made, self) + self._call_soon(self._protocol.connection_made, self) # only start reading when connection_made() has been called - self._loop.call_soon(self._add_reader, - self._sock_fd, self._read_ready) + self._call_soon(self._add_reader, self._sock_fd, self._read_ready) if waiter is not None: # only wake up the waiter when connection_made() has been called - self._loop.call_soon(futures._set_result_unless_cancelled, - waiter, None) + self._call_soon(futures._set_result_unless_cancelled, waiter, None) def set_protocol(self, protocol): if isinstance(protocol, protocols.BufferedProtocol): @@ -1081,10 +1088,11 @@ def write(self, data): if not data: return # Not all was written; register write handler. - self._loop._add_writer(self._sock_fd, self._write_ready) + self._add_writer(self._sock_fd, self._write_ready) # Add it to the buffer. self._buffer.append(data) + self._buffer_size += len(data) self._maybe_pause_protocol() def _get_sendmsg_buffer(self): @@ -1104,6 +1112,7 @@ def _write_sendmsg(self): except BaseException as exc: self._loop._remove_writer(self._sock_fd) self._buffer.clear() + self._buffer_size = 0 self._fatal_error(exc, 'Fatal write error on socket transport') if self._empty_waiter is not None: self._empty_waiter.set_exception(exc) @@ -1119,6 +1128,7 @@ def _write_sendmsg(self): self._sock.shutdown(socket.SHUT_WR) def _adjust_leftover_buffer(self, nbytes: int) -> None: + self._buffer_size -= nbytes buffer = self._buffer while nbytes: b = buffer.popleft() @@ -1139,13 +1149,16 @@ def _write_send(self): if n != len(buffer): # Not all data was written self._buffer.appendleft(buffer[n:]) + self._buffer_size -= n except (BlockingIOError, InterruptedError): - pass + self._buffer.appendleft(buffer) + return except (SystemExit, KeyboardInterrupt): raise except BaseException as exc: self._loop._remove_writer(self._sock_fd) self._buffer.clear() + self._buffer_size = 0 self._fatal_error(exc, 'Fatal write error on socket transport') if self._empty_waiter is not None: self._empty_waiter.set_exception(exc) @@ -1181,11 +1194,13 @@ def writelines(self, list_of_data): self._conn_lost += 1 return - self._buffer.extend([memoryview(data) for data in list_of_data]) + for data in list_of_data: + self._buffer.append(memoryview(data)) + self._buffer_size += len(data) self._write_ready() # If the entire buffer couldn't be written, register a write handler if self._buffer: - self._loop._add_writer(self._sock_fd, self._write_ready) + self._add_writer(self._sock_fd, self._write_ready) self._maybe_pause_protocol() def can_write_eof(self): @@ -1226,14 +1241,12 @@ def __init__(self, loop, sock, protocol, address=None, super().__init__(loop, sock, protocol, extra) self._address = address self._buffer_size = 0 - self._loop.call_soon(self._protocol.connection_made, self) + self._call_soon(self._protocol.connection_made, self) # only start reading when connection_made() has been called - self._loop.call_soon(self._add_reader, - self._sock_fd, self._read_ready) + self._call_soon(self._add_reader, self._sock_fd, self._read_ready) if waiter is not None: # only wake up the waiter when connection_made() has been called - self._loop.call_soon(futures._set_result_unless_cancelled, - waiter, None) + self._call_soon(futures._set_result_unless_cancelled, waiter, None) def get_write_buffer_size(self): return self._buffer_size @@ -1280,7 +1293,7 @@ def sendto(self, data, addr=None): self._sock.sendto(data, addr) return except (BlockingIOError, InterruptedError): - self._loop._add_writer(self._sock_fd, self._sendto_ready) + self._add_writer(self._sock_fd, self._sendto_ready) except OSError as exc: self._protocol.error_received(exc) return diff --git a/Lib/base64.py b/Lib/base64.py index a429760da79f2a..a94bec4d031c52 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -46,13 +46,15 @@ def _bytes_from_decode_data(s): # Base64 encoding/decoding uses binascii -def b64encode(s, altchars=None, *, wrapcol=0): +def b64encode(s, altchars=None, *, padded=True, wrapcol=0): """Encode the bytes-like object s using Base64 and return a bytes object. Optional altchars should be a byte string of length 2 which specifies an alternative alphabet for the '+' and '/' characters. This allows an application to e.g. generate url or filesystem safe Base64 strings. + If padded is false, omit padding in the output. + If wrapcol is non-zero, insert a newline (b'\\n') character after at most every wrapcol characters. """ @@ -60,18 +62,21 @@ def b64encode(s, altchars=None, *, wrapcol=0): if len(altchars) != 2: raise ValueError(f'invalid altchars: {altchars!r}') alphabet = binascii.BASE64_ALPHABET[:-2] + altchars - return binascii.b2a_base64(s, wrapcol=wrapcol, newline=False, + return binascii.b2a_base64(s, padded=padded, wrapcol=wrapcol, newline=False, alphabet=alphabet) - return binascii.b2a_base64(s, wrapcol=wrapcol, newline=False) + return binascii.b2a_base64(s, padded=padded, wrapcol=wrapcol, newline=False) -def b64decode(s, altchars=None, validate=_NOT_SPECIFIED, *, ignorechars=_NOT_SPECIFIED): +def b64decode(s, altchars=None, validate=_NOT_SPECIFIED, + *, padded=True, ignorechars=_NOT_SPECIFIED): """Decode the Base64 encoded bytes-like object or ASCII string s. Optional altchars must be a bytes-like object or ASCII string of length 2 which specifies the alternative alphabet used instead of the '+' and '/' characters. + If padded is false, padding in input is not required. + The result is returned as a bytes object. A binascii.Error is raised if s is incorrectly padded. @@ -105,11 +110,11 @@ def b64decode(s, altchars=None, validate=_NOT_SPECIFIED, *, ignorechars=_NOT_SPE alphabet = binascii.BASE64_ALPHABET[:-2] + altchars return binascii.a2b_base64(s, strict_mode=validate, alphabet=alphabet, - ignorechars=ignorechars) + padded=padded, ignorechars=ignorechars) if ignorechars is _NOT_SPECIFIED: ignorechars = b'' result = binascii.a2b_base64(s, strict_mode=validate, - ignorechars=ignorechars) + padded=padded, ignorechars=ignorechars) if badchar is not None: import warnings if validate: @@ -145,17 +150,19 @@ def standard_b64decode(s): _urlsafe_decode_translation = bytes.maketrans(b'-_', b'+/') -def urlsafe_b64encode(s): +def urlsafe_b64encode(s, *, padded=True): """Encode bytes using the URL- and filesystem-safe Base64 alphabet. Argument s is a bytes-like object to encode. The result is returned as a bytes object. The alphabet uses '-' instead of '+' and '_' instead of '/'. + + If padded is false, omit padding in the output. """ - return binascii.b2a_base64(s, newline=False, + return binascii.b2a_base64(s, padded=padded, newline=False, alphabet=binascii.URLSAFE_BASE64_ALPHABET) -def urlsafe_b64decode(s): +def urlsafe_b64decode(s, *, padded=False): """Decode bytes using the URL- and filesystem-safe Base64 alphabet. Argument s is a bytes-like object or ASCII string to decode. The result @@ -164,6 +171,8 @@ def urlsafe_b64decode(s): alphabet, and are not a plus '+' or slash '/', are discarded prior to the padding check. + If padded is false, padding in input is not required. + The alphabet uses '-' instead of '+' and '_' instead of '/'. """ s = _bytes_from_decode_data(s) @@ -173,7 +182,7 @@ def urlsafe_b64decode(s): badchar = b break s = s.translate(_urlsafe_decode_translation) - result = binascii.a2b_base64(s, strict_mode=False) + result = binascii.a2b_base64(s, strict_mode=False, padded=padded) if badchar is not None: import warnings warnings.warn(f'invalid character {chr(badchar)!a} in URL-safe Base64 data ' @@ -186,12 +195,22 @@ def urlsafe_b64decode(s): # Base32 encoding/decoding must be done in Python _B32_ENCODE_DOCSTRING = ''' Encode the bytes-like objects using {encoding} and return a bytes object. + +If padded is false, omit padding in the output. + +If wrapcol is non-zero, insert a newline (b'\\n') character after at most +every wrapcol characters. ''' _B32_DECODE_DOCSTRING = ''' Decode the {encoding} encoded bytes-like object or ASCII string s. Optional casefold is a flag specifying whether a lowercase alphabet is acceptable as input. For security purposes, the default is False. + +If padded is false, padding in input is not required. + +ignorechars should be a byte string containing characters to ignore +from the input. {extra_args} The result is returned as a bytes object. A binascii.Error is raised if the input is incorrectly padded or if there are non-alphabet @@ -206,54 +225,13 @@ def urlsafe_b64decode(s): the letter O). For security purposes the default is None, so that 0 and 1 are not allowed in the input. ''' -_b32alphabet = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567' -_b32hexalphabet = b'0123456789ABCDEFGHIJKLMNOPQRSTUV' -_b32tab2 = {} -_b32rev = {} - -def _b32encode(alphabet, s): - # Delay the initialization of the table to not waste memory - # if the function is never called - if alphabet not in _b32tab2: - b32tab = [bytes((i,)) for i in alphabet] - _b32tab2[alphabet] = [a + b for a in b32tab for b in b32tab] - b32tab = None - - if not isinstance(s, bytes_types): - s = memoryview(s).tobytes() - leftover = len(s) % 5 - # Pad the last quantum with zero bits if necessary - if leftover: - s = s + b'\0' * (5 - leftover) # Don't use += ! - encoded = bytearray() - from_bytes = int.from_bytes - b32tab2 = _b32tab2[alphabet] - for i in range(0, len(s), 5): - c = from_bytes(s[i: i + 5]) # big endian - encoded += (b32tab2[c >> 30] + # bits 1 - 10 - b32tab2[(c >> 20) & 0x3ff] + # bits 11 - 20 - b32tab2[(c >> 10) & 0x3ff] + # bits 21 - 30 - b32tab2[c & 0x3ff] # bits 31 - 40 - ) - # Adjust for any leftover partial quanta - if leftover == 1: - encoded[-6:] = b'======' - elif leftover == 2: - encoded[-4:] = b'====' - elif leftover == 3: - encoded[-3:] = b'===' - elif leftover == 4: - encoded[-1:] = b'=' - return encoded.take_bytes() - -def _b32decode(alphabet, s, casefold=False, map01=None): - # Delay the initialization of the table to not waste memory - # if the function is never called - if alphabet not in _b32rev: - _b32rev[alphabet] = {v: k for k, v in enumerate(alphabet)} + +def b32encode(s, *, padded=True, wrapcol=0): + return binascii.b2a_base32(s, padded=padded, wrapcol=wrapcol) +b32encode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32') + +def b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b''): s = _bytes_from_decode_data(s) - if len(s) % 8: - raise binascii.Error('Incorrect padding') # Handle section 2.4 zero and one mapping. The flag map01 will be either # False, or the character to map the digit 1 (one) to. It should be # either L (el) or I (eye). @@ -263,51 +241,22 @@ def _b32decode(alphabet, s, casefold=False, map01=None): s = s.translate(bytes.maketrans(b'01', b'O' + map01)) if casefold: s = s.upper() - # Strip off pad characters from the right. We need to count the pad - # characters because this will tell us how many null bytes to remove from - # the end of the decoded string. - l = len(s) - s = s.rstrip(b'=') - padchars = l - len(s) - # Now decode the full quanta - decoded = bytearray() - b32rev = _b32rev[alphabet] - for i in range(0, len(s), 8): - quanta = s[i: i + 8] - acc = 0 - try: - for c in quanta: - acc = (acc << 5) + b32rev[c] - except KeyError: - raise binascii.Error('Non-base32 digit found') from None - decoded += acc.to_bytes(5) # big endian - # Process the last, partial quanta - if l % 8 or padchars not in {0, 1, 3, 4, 6}: - raise binascii.Error('Incorrect padding') - if padchars and decoded: - acc <<= 5 * padchars - last = acc.to_bytes(5) # big endian - leftover = (43 - 5 * padchars) // 8 # 1: 4, 3: 3, 4: 2, 6: 1 - decoded[-5:] = last[:leftover] - return decoded.take_bytes() - - -def b32encode(s): - return _b32encode(_b32alphabet, s) -b32encode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32') - -def b32decode(s, casefold=False, map01=None): - return _b32decode(_b32alphabet, s, casefold, map01) + return binascii.a2b_base32(s, padded=padded, ignorechars=ignorechars) b32decode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32', extra_args=_B32_DECODE_MAP01_DOCSTRING) -def b32hexencode(s): - return _b32encode(_b32hexalphabet, s) +def b32hexencode(s, *, padded=True, wrapcol=0): + return binascii.b2a_base32(s, padded=padded, wrapcol=wrapcol, + alphabet=binascii.BASE32HEX_ALPHABET) b32hexencode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32hex') -def b32hexdecode(s, casefold=False): +def b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b''): + s = _bytes_from_decode_data(s) # base32hex does not have the 01 mapping - return _b32decode(_b32hexalphabet, s, casefold) + if casefold: + s = s.upper() + return binascii.a2b_base32(s, alphabet=binascii.BASE32HEX_ALPHABET, + padded=padded, ignorechars=ignorechars) b32hexdecode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32hex', extra_args='') @@ -315,28 +264,43 @@ def b32hexdecode(s, casefold=False): # RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns # lowercase. The RFC also recommends against accepting input case # insensitively. -def b16encode(s): +def b16encode(s, *, wrapcol=0): """Encode the bytes-like object s using Base16 and return a bytes object. + + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. """ - return binascii.hexlify(s).upper() + if not wrapcol: + return binascii.hexlify(s).upper() + if wrapcol < 0: + raise ValueError('Negative wrapcol') + if wrapcol < 2: + wrapcol = 2 + return binascii.hexlify(s, bytes_per_sep=-(wrapcol//2), sep=b'\n').upper() -def b16decode(s, casefold=False): +def b16decode(s, casefold=False, *, ignorechars=b''): """Decode the Base16 encoded bytes-like object or ASCII string s. Optional casefold is a flag specifying whether a lowercase alphabet is acceptable as input. For security purposes, the default is False. + ignorechars should be a byte string containing characters to ignore + from the input. + The result is returned as a bytes object. A binascii.Error is raised if s is incorrectly padded or if there are non-alphabet characters present in the input. """ - s = _bytes_from_decode_data(s) - if casefold: - s = s.upper() - if s.translate(None, delete=b'0123456789ABCDEF'): - raise binascii.Error('Non-base16 digit found') - return binascii.unhexlify(s) + if not casefold: + s = _bytes_from_decode_data(s) + if not isinstance(ignorechars, bytes): + ignorechars = bytes(memoryview(ignorechars)) + for b in b'abcdef': + if b in s and b not in ignorechars: + raise binascii.Error('Non-base16 digit found') + s = s.translate(None, delete=b'abcdef') + return binascii.unhexlify(s, ignorechars=ignorechars) # # Ascii85 encoding/decoding @@ -379,31 +343,42 @@ def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'): return binascii.a2b_ascii85(b, foldspaces=foldspaces, adobe=adobe, ignorechars=ignorechars) -def b85encode(b, pad=False): +def b85encode(b, pad=False, *, wrapcol=0): """Encode bytes-like object b in base85 format and return a bytes object. + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. + If pad is true, the input is padded with b'\\0' so its length is a multiple of 4 bytes before encoding. """ - return binascii.b2a_base85(b, pad=pad) + return binascii.b2a_base85(b, wrapcol=wrapcol, pad=pad) -def b85decode(b): +def b85decode(b, *, ignorechars=b''): """Decode the base85-encoded bytes-like object or ASCII string b The result is returned as a bytes object. """ - return binascii.a2b_base85(b) + return binascii.a2b_base85(b, ignorechars=ignorechars) + +def z85encode(s, pad=False, *, wrapcol=0): + """Encode bytes-like object b in z85 format and return a bytes object. + + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. -def z85encode(s, pad=False): - """Encode bytes-like object b in z85 format and return a bytes object.""" - return binascii.b2a_base85(s, pad=pad, alphabet=binascii.Z85_ALPHABET) + If pad is true, the input is padded with b'\\0' so its length is a multiple of + 4 bytes before encoding. + """ + return binascii.b2a_base85(s, wrapcol=wrapcol, pad=pad, + alphabet=binascii.Z85_ALPHABET) -def z85decode(s): +def z85decode(s, *, ignorechars=b''): """Decode the z85-encoded bytes-like object or ASCII string b The result is returned as a bytes object. """ - return binascii.a2b_base85(s, alphabet=binascii.Z85_ALPHABET) + return binascii.a2b_base85(s, alphabet=binascii.Z85_ALPHABET, ignorechars=ignorechars) # Legacy interface. This code could be cleaned up since I don't believe # binascii has any line length limitations. It just doesn't seem worth it diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 2eee4c70955513..febab521629228 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -328,14 +328,14 @@ def __ior__(self, other): return self def __or__(self, other): - if not isinstance(other, dict): + if not isinstance(other, (dict, frozendict)): return NotImplemented new = self.__class__(self) new.update(other) return new def __ror__(self, other): - if not isinstance(other, dict): + if not isinstance(other, (dict, frozendict)): return NotImplemented new = self.__class__(other) new.update(self) @@ -1216,14 +1216,14 @@ def __repr__(self): def __or__(self, other): if isinstance(other, UserDict): return self.__class__(self.data | other.data) - if isinstance(other, dict): + if isinstance(other, (dict, frozendict)): return self.__class__(self.data | other) return NotImplemented def __ror__(self, other): if isinstance(other, UserDict): return self.__class__(other.data | self.data) - if isinstance(other, dict): + if isinstance(other, (dict, frozendict)): return self.__class__(other | self.data) return NotImplemented diff --git a/Lib/curses/has_key.py b/Lib/curses/has_key.py index 4e37b480f13353..3471b28cbbe017 100644 --- a/Lib/curses/has_key.py +++ b/Lib/curses/has_key.py @@ -7,7 +7,7 @@ # Table mapping curses keys to the terminfo capability name -_capability_names = { +_capability_names = frozendict({ _curses.KEY_A1: 'ka1', _curses.KEY_A3: 'ka3', _curses.KEY_B2: 'kb2', @@ -157,7 +157,7 @@ _curses.KEY_SUSPEND: 'kspd', _curses.KEY_UNDO: 'kund', _curses.KEY_UP: 'kcuu1' - } + }) def has_key(ch): if isinstance(ch, str): @@ -170,7 +170,7 @@ def has_key(ch): #Check the current terminal description for that capability; #if present, return true, else return false. - if _curses.tigetstr( capability_name ): + if _curses.tigetstr(capability_name): return True else: return False diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 6164ea62324cce..93b4e7a820f3ad 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -16,7 +16,8 @@ # policies recommend against bundling dependencies. For example, Fedora # installs wheel packages in the /usr/share/python-wheels/ directory and don't # install the ensurepip._bundled package. -if (_pkg_dir := sysconfig.get_config_var('WHEEL_PKG_DIR')) is not None: +_pkg_dir = sysconfig.get_config_var('WHEEL_PKG_DIR') +if _pkg_dir: _WHEEL_PKG_DIR = Path(_pkg_dir).resolve() else: _WHEEL_PKG_DIR = None diff --git a/Lib/getpass.py b/Lib/getpass.py index 3d9bb1f0d146a4..cfbd63dded6cc1 100644 --- a/Lib/getpass.py +++ b/Lib/getpass.py @@ -26,6 +26,45 @@ class GetPassWarning(UserWarning): pass +# Default POSIX control character mappings +_POSIX_CTRL_CHARS = frozendict({ + 'BS': '\x08', # Backspace + 'ERASE': '\x7f', # DEL + 'KILL': '\x15', # Ctrl+U - kill line + 'WERASE': '\x17', # Ctrl+W - erase word + 'LNEXT': '\x16', # Ctrl+V - literal next + 'EOF': '\x04', # Ctrl+D - EOF + 'INTR': '\x03', # Ctrl+C - interrupt + 'SOH': '\x01', # Ctrl+A - start of heading (beginning of line) + 'ENQ': '\x05', # Ctrl+E - enquiry (end of line) + 'VT': '\x0b', # Ctrl+K - vertical tab (kill forward) +}) + + +def _get_terminal_ctrl_chars(fd): + """Extract control characters from terminal settings. + + Returns a dict mapping control char names to their str values. + + Falls back to POSIX defaults if termios is not available + or if the control character is not supported by termios. + """ + ctrl = dict(_POSIX_CTRL_CHARS) + try: + old = termios.tcgetattr(fd) + cc = old[6] # Index 6 is the control characters array + except (termios.error, OSError): + return ctrl + + # Use defaults for Backspace (BS) and Ctrl+A/E/K (SOH/ENQ/VT) + # as they are not in the termios control characters array. + for name in ('ERASE', 'KILL', 'WERASE', 'LNEXT', 'EOF', 'INTR'): + cap = getattr(termios, f'V{name}') + if cap < len(cc): + ctrl[name] = cc[cap].decode('latin-1') + return ctrl + + def unix_getpass(prompt='Password: ', stream=None, *, echo_char=None): """Prompt for a password, with echo turned off. @@ -73,15 +112,27 @@ def unix_getpass(prompt='Password: ', stream=None, *, echo_char=None): old = termios.tcgetattr(fd) # a copy to save new = old[:] new[3] &= ~termios.ECHO # 3 == 'lflags' + # Extract control characters before changing terminal mode. + term_ctrl_chars = None if echo_char: + # ICANON enables canonical (line-buffered) mode where + # the terminal handles line editing. Disable it so we + # can read input char by char and handle editing ourselves. new[3] &= ~termios.ICANON + # IEXTEN enables implementation-defined input processing + # such as LNEXT (Ctrl+V). Disable it so the terminal + # driver does not intercept these characters before our + # code can handle them. + new[3] &= ~termios.IEXTEN + term_ctrl_chars = _get_terminal_ctrl_chars(fd) tcsetattr_flags = termios.TCSAFLUSH if hasattr(termios, 'TCSASOFT'): tcsetattr_flags |= termios.TCSASOFT try: termios.tcsetattr(fd, tcsetattr_flags, new) passwd = _raw_input(prompt, stream, input=input, - echo_char=echo_char) + echo_char=echo_char, + term_ctrl_chars=term_ctrl_chars) finally: termios.tcsetattr(fd, tcsetattr_flags, old) @@ -159,7 +210,8 @@ def _check_echo_char(echo_char): f"character, got: {echo_char!r}") -def _raw_input(prompt="", stream=None, input=None, echo_char=None): +def _raw_input(prompt="", stream=None, input=None, echo_char=None, + term_ctrl_chars=None): # This doesn't save the string in the GNU readline history. if not stream: stream = sys.stderr @@ -177,7 +229,8 @@ def _raw_input(prompt="", stream=None, input=None, echo_char=None): stream.flush() # NOTE: The Python C API calls flockfile() (and unlock) during readline. if echo_char: - return _readline_with_echo_char(stream, input, echo_char) + return _readline_with_echo_char(stream, input, echo_char, + term_ctrl_chars, prompt) line = input.readline() if not line: raise EOFError @@ -186,33 +239,174 @@ def _raw_input(prompt="", stream=None, input=None, echo_char=None): return line -def _readline_with_echo_char(stream, input, echo_char): - passwd = "" - eof_pressed = False - while True: - char = input.read(1) - if char == '\n' or char == '\r': - break - elif char == '\x03': - raise KeyboardInterrupt - elif char == '\x7f' or char == '\b': - if passwd: - stream.write("\b \b") - stream.flush() - passwd = passwd[:-1] - elif char == '\x04': - if eof_pressed: +def _readline_with_echo_char(stream, input, echo_char, term_ctrl_chars=None, + prompt=""): + """Read password with echo character and line editing support.""" + if term_ctrl_chars is None: + term_ctrl_chars = _POSIX_CTRL_CHARS + + editor = _PasswordLineEditor(stream, echo_char, term_ctrl_chars, prompt) + return editor.readline(input) + + +class _PasswordLineEditor: + """Handles line editing for password input with echo character.""" + + def __init__(self, stream, echo_char, ctrl_chars, prompt=""): + self.stream = stream + self.echo_char = echo_char + self.prompt = prompt + self.password = [] + self.cursor_pos = 0 + self.eof_pressed = False + self.literal_next = False + self.ctrl = ctrl_chars + self.dispatch = { + ctrl_chars['SOH']: self.handle_move_start, # Ctrl+A + ctrl_chars['ENQ']: self.handle_move_end, # Ctrl+E + ctrl_chars['VT']: self.handle_kill_forward, # Ctrl+K + ctrl_chars['KILL']: self.handle_kill_line, # Ctrl+U + ctrl_chars['WERASE']: self.handle_erase_word, # Ctrl+W + ctrl_chars['ERASE']: self.handle_erase, # DEL + ctrl_chars['BS']: self.handle_erase, # Backspace + # special characters + ctrl_chars['LNEXT']: self.handle_literal_next, # Ctrl+V + ctrl_chars['EOF']: self.handle_eof, # Ctrl+D + ctrl_chars['INTR']: self.handle_interrupt, # Ctrl+C + '\x00': self.handle_nop, # ignore NUL + } + + def refresh_display(self, prev_len=None): + """Redraw the entire password line with *echo_char*. + + If *prev_len* is not specified, the current password length is used. + """ + prompt_len = len(self.prompt) + clear_len = prev_len if prev_len is not None else len(self.password) + # Clear the entire line (prompt + password) and rewrite. + self.stream.write('\r' + ' ' * (prompt_len + clear_len) + '\r') + self.stream.write(self.prompt + self.echo_char * len(self.password)) + if self.cursor_pos < len(self.password): + self.stream.write('\b' * (len(self.password) - self.cursor_pos)) + self.stream.flush() + + def insert_char(self, char): + """Insert *char* at cursor position.""" + self.password.insert(self.cursor_pos, char) + self.cursor_pos += 1 + # Only refresh if inserting in middle. + if self.cursor_pos < len(self.password): + self.refresh_display() + else: + self.stream.write(self.echo_char) + self.stream.flush() + + def is_eol(self, char): + """Check if *char* is a line terminator.""" + return char in ('\r', '\n') + + def is_eof(self, char): + """Check if *char* is a file terminator.""" + return char == self.ctrl['EOF'] + + def handle_move_start(self): + """Move cursor to beginning (Ctrl+A).""" + self.cursor_pos = 0 + self.refresh_display() + + def handle_move_end(self): + """Move cursor to end (Ctrl+E).""" + self.cursor_pos = len(self.password) + self.refresh_display() + + def handle_erase(self): + """Delete character before cursor (Backspace/DEL).""" + if self.cursor_pos == 0: + return + assert self.cursor_pos > 0 + self.cursor_pos -= 1 + prev_len = len(self.password) + del self.password[self.cursor_pos] + self.refresh_display(prev_len) + + def handle_kill_line(self): + """Erase entire line (Ctrl+U).""" + prev_len = len(self.password) + self.password.clear() + self.cursor_pos = 0 + self.refresh_display(prev_len) + + def handle_kill_forward(self): + """Kill from cursor to end (Ctrl+K).""" + prev_len = len(self.password) + del self.password[self.cursor_pos:] + self.refresh_display(prev_len) + + def handle_erase_word(self): + """Erase previous word (Ctrl+W).""" + old_cursor = self.cursor_pos + # Calculate the starting position of the previous word, + # ignoring trailing whitespaces. + while self.cursor_pos > 0 and self.password[self.cursor_pos - 1] == ' ': + self.cursor_pos -= 1 + while self.cursor_pos > 0 and self.password[self.cursor_pos - 1] != ' ': + self.cursor_pos -= 1 + # Delete the previous word and refresh the screen. + prev_len = len(self.password) + del self.password[self.cursor_pos:old_cursor] + self.refresh_display(prev_len) + + def handle_literal_next(self): + """State transition to indicate that the next character is literal.""" + assert self.literal_next is False + self.literal_next = True + + def handle_eof(self): + """State transition to indicate that the pressed character was EOF.""" + assert self.eof_pressed is False + self.eof_pressed = True + + def handle_interrupt(self): + """Raise a KeyboardInterrupt after Ctrl+C has been received.""" + raise KeyboardInterrupt + + def handle_nop(self): + """Handler for an ignored character.""" + + def handle(self, char): + """Handle a single character input. Returns True if handled.""" + handler = self.dispatch.get(char) + if handler: + handler() + return True + return False + + def readline(self, input): + """Read a line of password input with echo character support.""" + while True: + assert self.cursor_pos >= 0 + char = input.read(1) + if self.is_eol(char): break + # Handle literal next mode first as Ctrl+V quotes characters. + elif self.literal_next: + self.insert_char(char) + self.literal_next = False + # Handle EOF now as Ctrl+D must be pressed twice + # consecutively to stop reading from the input. + elif self.is_eof(char): + if self.eof_pressed: + break + elif self.handle(char): + # Dispatched to handler. + pass else: - eof_pressed = True - elif char == '\x00': - continue - else: - passwd += char - stream.write(echo_char) - stream.flush() - eof_pressed = False - return passwd + # Insert as normal character. + self.insert_char(char) + + self.eof_pressed = self.is_eof(char) + + return ''.join(self.password) def getuser(): diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py index e91acc065ba9ae..32f4b7d2d6e08b 100644 --- a/Lib/importlib/metadata/__init__.py +++ b/Lib/importlib/metadata/__init__.py @@ -31,7 +31,8 @@ from . import _meta from ._collections import FreezableDefaultDict, Pair -from ._functools import method_cache, pass_none +from ._context import ExceptionTrap +from ._functools import method_cache, noop, pass_none, passthrough from ._itertools import always_iterable, bucket, unique_everseen from ._meta import PackageMetadata, SimplePath from ._typing import md_none @@ -42,6 +43,7 @@ 'PackageMetadata', 'PackageNotFoundError', 'PackagePath', + 'MetadataNotFound', 'SimplePath', 'distribution', 'distributions', @@ -66,6 +68,10 @@ def name(self) -> str: # type: ignore[override] # make readonly return name +class MetadataNotFound(FileNotFoundError): + """No metadata file is present in the distribution.""" + + class Sectioned: """ A simple entry point config parser for performance @@ -487,7 +493,12 @@ def _prefer_valid(dists: Iterable[Distribution]) -> Iterable[Distribution]: Ref python/importlib_resources#489. """ - buckets = bucket(dists, lambda dist: bool(dist.metadata)) + + has_metadata = ExceptionTrap(MetadataNotFound).passes( + operator.attrgetter('metadata') + ) + + buckets = bucket(dists, has_metadata) return itertools.chain(buckets[True], buckets[False]) @staticmethod @@ -508,7 +519,7 @@ def _discover_resolvers(): return filter(None, declared) @property - def metadata(self) -> _meta.PackageMetadata | None: + def metadata(self) -> _meta.PackageMetadata: """Return the parsed metadata for this Distribution. The returned object will have keys that name the various bits of @@ -517,6 +528,8 @@ def metadata(self) -> _meta.PackageMetadata | None: Custom providers may provide the METADATA file or override this property. + + :raises MetadataNotFound: If no metadata file is present. """ text = ( @@ -527,20 +540,25 @@ def metadata(self) -> _meta.PackageMetadata | None: # (which points to the egg-info file) attribute unchanged. or self.read_text('') ) - return self._assemble_message(text) + return self._assemble_message(self._ensure_metadata_present(text)) @staticmethod - @pass_none def _assemble_message(text: str) -> _meta.PackageMetadata: # deferred for performance (python/cpython#109829) from . import _adapters return _adapters.Message(email.message_from_string(text)) + def _ensure_metadata_present(self, text: str | None) -> str: + if text is not None: + return text + + raise MetadataNotFound('No package metadata was found.') + @property def name(self) -> str: """Return the 'Name' metadata for the distribution package.""" - return md_none(self.metadata)['Name'] + return self.metadata['Name'] @property def _normalized_name(self): @@ -550,7 +568,7 @@ def _normalized_name(self): @property def version(self) -> str: """Return the 'Version' metadata for the distribution package.""" - return md_none(self.metadata)['Version'] + return self.metadata['Version'] @property def entry_points(self) -> EntryPoints: @@ -783,6 +801,20 @@ def find_distributions(self, context=Context()) -> Iterable[Distribution]: """ +@passthrough +def _clear_after_fork(cached): + """Ensure ``func`` clears cached state after ``fork`` when supported. + + ``FastPath`` caches zip-backed ``pathlib.Path`` objects that retain a + reference to the parent's open ``ZipFile`` handle. Re-using a cached + instance in a forked child can therefore resurrect invalid file pointers + and trigger ``BadZipFile``/``OSError`` failures (python/importlib_metadata#520). + Registering ``cache_clear`` with ``os.register_at_fork`` keeps each process + on its own cache. + """ + getattr(os, 'register_at_fork', noop)(after_in_child=cached.cache_clear) + + class FastPath: """ Micro-optimized class for searching a root for children. @@ -799,7 +831,8 @@ class FastPath: True """ - @functools.lru_cache() # type: ignore[misc] + @_clear_after_fork # type: ignore[misc] + @functools.lru_cache() def __new__(cls, root): return super().__new__(cls) @@ -925,10 +958,12 @@ def __init__(self, name: str | None): def normalize(name): """ PEP 503 normalization plus dashes as underscores. + + Specifically avoids ``re.sub`` as prescribed for performance + benefits (see python/cpython#143658). """ - # Much faster than re.sub, and even faster than str.translate value = name.lower().replace("-", "_").replace(".", "_") - # Condense repeats (faster than regex) + # Condense repeats while "__" in value: value = value.replace("__", "_") return value @@ -1046,11 +1081,12 @@ def distributions(**kwargs) -> Iterable[Distribution]: return Distribution.discover(**kwargs) -def metadata(distribution_name: str) -> _meta.PackageMetadata | None: +def metadata(distribution_name: str) -> _meta.PackageMetadata: """Get the metadata for the named package. :param distribution_name: The name of the distribution package to query. :return: A PackageMetadata containing the parsed metadata. + :raises MetadataNotFound: If no metadata file is present in the distribution. """ return Distribution.from_name(distribution_name).metadata @@ -1121,7 +1157,7 @@ def packages_distributions() -> Mapping[str, list[str]]: pkg_to_dist = collections.defaultdict(list) for dist in distributions(): for pkg in _top_level_declared(dist) or _top_level_inferred(dist): - pkg_to_dist[pkg].append(md_none(dist.metadata)['Name']) + pkg_to_dist[pkg].append(dist.metadata['Name']) return dict(pkg_to_dist) diff --git a/Lib/importlib/metadata/_adapters.py b/Lib/importlib/metadata/_adapters.py index f5b30dd92cde69..dede395d79a38b 100644 --- a/Lib/importlib/metadata/_adapters.py +++ b/Lib/importlib/metadata/_adapters.py @@ -9,7 +9,8 @@ class RawPolicy(email.policy.EmailPolicy): def fold(self, name, value): folded = self.linesep.join( - textwrap.indent(value, prefix=' ' * 8, predicate=lambda line: True) + textwrap + .indent(value, prefix=' ' * 8, predicate=lambda line: True) .lstrip() .splitlines() ) diff --git a/Lib/importlib/metadata/_context.py b/Lib/importlib/metadata/_context.py new file mode 100644 index 00000000000000..2635b164ce8923 --- /dev/null +++ b/Lib/importlib/metadata/_context.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import functools +import operator + + +# from jaraco.context 6.1 +class ExceptionTrap: + """ + A context manager that will catch certain exceptions and provide an + indication they occurred. + + >>> with ExceptionTrap() as trap: + ... raise Exception() + >>> bool(trap) + True + + >>> with ExceptionTrap() as trap: + ... pass + >>> bool(trap) + False + + >>> with ExceptionTrap(ValueError) as trap: + ... raise ValueError("1 + 1 is not 3") + >>> bool(trap) + True + >>> trap.value + ValueError('1 + 1 is not 3') + >>> trap.tb + + + >>> with ExceptionTrap(ValueError) as trap: + ... raise Exception() + Traceback (most recent call last): + ... + Exception + + >>> bool(trap) + False + """ + + exc_info = None, None, None + + def __init__(self, exceptions=(Exception,)): + self.exceptions = exceptions + + def __enter__(self): + return self + + @property + def type(self): + return self.exc_info[0] + + @property + def value(self): + return self.exc_info[1] + + @property + def tb(self): + return self.exc_info[2] + + def __exit__(self, *exc_info): + type = exc_info[0] + matches = type and issubclass(type, self.exceptions) + if matches: + self.exc_info = exc_info + return matches + + def __bool__(self): + return bool(self.type) + + def raises(self, func, *, _test=bool): + """ + Wrap func and replace the result with the truth + value of the trap (True if an exception occurred). + + First, give the decorator an alias to support Python 3.8 + Syntax. + + >>> raises = ExceptionTrap(ValueError).raises + + Now decorate a function that always fails. + + >>> @raises + ... def fail(): + ... raise ValueError('failed') + >>> fail() + True + """ + + @functools.wraps(func) + def wrapper(*args, **kwargs): + with ExceptionTrap(self.exceptions) as trap: + func(*args, **kwargs) + return _test(trap) + + return wrapper + + def passes(self, func): + """ + Wrap func and replace the result with the truth + value of the trap (True if no exception). + + First, give the decorator an alias to support Python 3.8 + Syntax. + + >>> passes = ExceptionTrap(ValueError).passes + + Now decorate a function that always fails. + + >>> @passes + ... def fail(): + ... raise ValueError('failed') + + >>> fail() + False + """ + return self.raises(func, _test=operator.not_) diff --git a/Lib/importlib/metadata/_functools.py b/Lib/importlib/metadata/_functools.py index 5dda6a2199ad0b..c159b46e48959c 100644 --- a/Lib/importlib/metadata/_functools.py +++ b/Lib/importlib/metadata/_functools.py @@ -1,5 +1,7 @@ import functools import types +from collections.abc import Callable +from typing import TypeVar # from jaraco.functools 3.3 @@ -102,3 +104,33 @@ def wrapper(param, *args, **kwargs): return func(param, *args, **kwargs) return wrapper + + +# From jaraco.functools 4.4 +def noop(*args, **kwargs): + """ + A no-operation function that does nothing. + + >>> noop(1, 2, three=3) + """ + + +_T = TypeVar('_T') + + +# From jaraco.functools 4.4 +def passthrough(func: Callable[..., object]) -> Callable[[_T], _T]: + """ + Wrap the function to always return the first parameter. + + >>> passthrough(print)('3') + 3 + '3' + """ + + @functools.wraps(func) + def wrapper(first: _T, *args, **kwargs) -> _T: + func(first, *args, **kwargs) + return first + + return wrapper # type: ignore[return-value] diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index 89396b25a2cbb3..251025efac14b8 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -241,7 +241,7 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, **kw).encode(obj) -_default_decoder = JSONDecoder(object_hook=None, object_pairs_hook=None) +_default_decoder = JSONDecoder() def detect_encoding(b): @@ -275,7 +275,8 @@ def detect_encoding(b): def load(fp, *, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, **kw): + parse_int=None, parse_constant=None, object_pairs_hook=None, + array_hook=None, **kw): """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing a JSON document) to a Python object. @@ -291,17 +292,26 @@ def load(fp, *, cls=None, object_hook=None, parse_float=None, ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. + ``array_hook`` is an optional function that will be called with the result + of any literal array decode (a ``list``). The return value of this function will + be used instead of the ``list``. This feature can be used along + ``object_pairs_hook`` to customize the resulting data structure - for example, + by setting that to ``frozendict`` and ``array_hook`` to ``tuple``, one can get + a deep immutable data structute from any JSON data. + To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg; otherwise ``JSONDecoder`` is used. """ return loads(fp.read(), cls=cls, object_hook=object_hook, parse_float=parse_float, parse_int=parse_int, - parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, **kw) + parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, + array_hook=None, **kw) def loads(s, *, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, **kw): + parse_int=None, parse_constant=None, object_pairs_hook=None, + array_hook=None, **kw): """Deserialize ``s`` (a ``str``, ``bytes`` or ``bytearray`` instance containing a JSON document) to a Python object. @@ -317,6 +327,13 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None, ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. + ``array_hook`` is an optional function that will be called with the result + of any literal array decode (a ``list``). The return value of this function will + be used instead of the ``list``. This feature can be used along + ``object_pairs_hook`` to customize the resulting data structure - for example, + by setting that to ``frozendict`` and ``array_hook`` to ``tuple``, one can get + a deep immutable data structute from any JSON data. + ``parse_float``, if specified, will be called with the string of every JSON float to be decoded. By default this is equivalent to float(num_str). This can be used to use another datatype or parser @@ -347,7 +364,8 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None, if (cls is None and object_hook is None and parse_int is None and parse_float is None and - parse_constant is None and object_pairs_hook is None and not kw): + parse_constant is None and object_pairs_hook is None + and array_hook is None and not kw): return _default_decoder.decode(s) if cls is None: cls = JSONDecoder @@ -355,6 +373,8 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None, kw['object_hook'] = object_hook if object_pairs_hook is not None: kw['object_pairs_hook'] = object_pairs_hook + if array_hook is not None: + kw['array_hook'] = array_hook if parse_float is not None: kw['parse_float'] = parse_float if parse_int is not None: diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py index 4cd6f8367a1349..364e44d40cc307 100644 --- a/Lib/json/decoder.py +++ b/Lib/json/decoder.py @@ -218,7 +218,7 @@ def JSONObject(s_and_end, strict, scan_once, object_hook, object_pairs_hook, pairs = object_hook(pairs) return pairs, end -def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): +def JSONArray(s_and_end, scan_once, array_hook, _w=WHITESPACE.match, _ws=WHITESPACE_STR): s, end = s_and_end values = [] nextchar = s[end:end + 1] @@ -227,6 +227,8 @@ def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): nextchar = s[end:end + 1] # Look-ahead for trivial empty array if nextchar == ']': + if array_hook is not None: + values = array_hook(values) return values, end + 1 _append = values.append while True: @@ -256,6 +258,8 @@ def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): if nextchar == ']': raise JSONDecodeError("Illegal trailing comma before end of array", s, comma_idx) + if array_hook is not None: + values = array_hook(values) return values, end @@ -291,7 +295,7 @@ class JSONDecoder(object): def __init__(self, *, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, strict=True, - object_pairs_hook=None): + object_pairs_hook=None, array_hook=None): """``object_hook``, if specified, will be called with the result of every JSON object decoded and its return value will be used in place of the given ``dict``. This can be used to provide custom @@ -304,6 +308,14 @@ def __init__(self, *, object_hook=None, parse_float=None, If ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. + ``array_hook`` is an optional function that will be called with the + result of any literal array decode (a ``list``). The return value of + this function will be used instead of the ``list``. This feature can + be used along ``object_pairs_hook`` to customize the resulting data + structure - for example, by setting that to ``frozendict`` and + ``array_hook`` to ``tuple``, one can get a deep immutable data + structute from any JSON data. + ``parse_float``, if specified, will be called with the string of every JSON float to be decoded. By default this is equivalent to float(num_str). This can be used to use another datatype or parser @@ -330,6 +342,7 @@ def __init__(self, *, object_hook=None, parse_float=None, self.parse_constant = parse_constant or _CONSTANTS.__getitem__ self.strict = strict self.object_pairs_hook = object_pairs_hook + self.array_hook = array_hook self.parse_object = JSONObject self.parse_array = JSONArray self.parse_string = scanstring diff --git a/Lib/json/scanner.py b/Lib/json/scanner.py index 090897515fe2f3..b484e00be0fd2a 100644 --- a/Lib/json/scanner.py +++ b/Lib/json/scanner.py @@ -23,6 +23,7 @@ def py_make_scanner(context): parse_constant = context.parse_constant object_hook = context.object_hook object_pairs_hook = context.object_pairs_hook + array_hook = context.array_hook memo = context.memo def _scan_once(string, idx): @@ -37,7 +38,7 @@ def _scan_once(string, idx): return parse_object((string, idx + 1), strict, _scan_once, object_hook, object_pairs_hook, memo) elif nextchar == '[': - return parse_array((string, idx + 1), _scan_once) + return parse_array((string, idx + 1), _scan_once, array_hook) elif nextchar == 'n' and string[idx:idx + 4] == 'null': return None, idx + 4 elif nextchar == 't' and string[idx:idx + 4] == 'true': diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 39689a57e6ecd6..6eef90ae5cd914 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -622,6 +622,9 @@ def __init__(self, fmt=None, datefmt=None, style='%', validate=True, *, self._fmt = self._style._fmt self.datefmt = datefmt + def __repr__(self): + return '<%s (%s)>' % (self.__class__.__name__, self._fmt) + default_time_format = '%Y-%m-%d %H:%M:%S' default_msec_format = '%s,%03d' @@ -794,6 +797,9 @@ def __init__(self, name=''): self.name = name self.nlen = len(name) + def __repr__(self): + return '<%s (%s)>' % (self.__class__.__name__, self.name) + def filter(self, record): """ Determine if the specified record is to be logged. diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 60e8c2be1e2504..a834826114614d 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -478,6 +478,7 @@ def _default_mime_types(): '.js' : 'text/javascript', '.mjs' : 'text/javascript', '.dcm' : 'application/dicom', + '.efi' : 'application/efi', '.epub' : 'application/epub+zip', '.gz' : 'application/gzip', '.json' : 'application/json', diff --git a/Lib/netrc.py b/Lib/netrc.py index 750b5071e3c65f..a28ea297df894b 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -152,23 +152,28 @@ def _parse(self, file, fp, default_netrc): else: raise NetrcParseError("bad follower token %r" % tt, file, lexer.lineno) - self._security_check(fp, default_netrc, self.hosts[entryname][0]) - - def _security_check(self, fp, default_netrc, login): - if _can_security_check() and default_netrc and login != "anonymous": - prop = os.fstat(fp.fileno()) - current_user_id = os.getuid() - if prop.st_uid != current_user_id: - fowner = _getpwuid(prop.st_uid) - user = _getpwuid(current_user_id) - raise NetrcParseError( - f"~/.netrc file owner ({fowner}) does not match" - f" current user ({user})") - if (prop.st_mode & (stat.S_IRWXG | stat.S_IRWXO)): - raise NetrcParseError( - "~/.netrc access too permissive: access" - " permissions must restrict access to only" - " the owner") + + if _can_security_check() and default_netrc: + for entry in self.hosts.values(): + if entry[0] != "anonymous": + # Raises on security issue; once passed once can exit. + self._security_check(fp) + return + + def _security_check(self, fp): + prop = os.fstat(fp.fileno()) + current_user_id = os.getuid() + if prop.st_uid != current_user_id: + fowner = _getpwuid(prop.st_uid) + user = _getpwuid(current_user_id) + raise NetrcParseError( + f"~/.netrc file owner ({fowner}) does not match" + f" current user ({user})") + if (prop.st_mode & (stat.S_IRWXG | stat.S_IRWXO)): + raise NetrcParseError( + "~/.netrc access too permissive: access" + " permissions must restrict access to only" + " the owner") def authenticators(self, host): """Return a (user, account, password) tuple for given host.""" diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 3c6a6b7bdc44d2..93f3ef5e38af84 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -2,7 +2,7 @@ The property list (.plist) file format is a simple XML pickle supporting basic object types, like dictionaries, lists, numbers and strings. -Usually the top level object is a dictionary. +Usually the top level object is a dictionary or a frozen dictionary. To write out a plist file, use the dump(value, file) function. 'value' is the top level object, 'file' is @@ -21,7 +21,7 @@ Generate Plist example: - import datetime + import datetime as dt import plistlib pl = dict( @@ -37,7 +37,7 @@ ), someData = b"", someMoreData = b"" * 10, - aDate = datetime.datetime.now() + aDate = dt.datetime.now() ) print(plistlib.dumps(pl).decode()) @@ -357,7 +357,7 @@ def write_value(self, value): elif isinstance(value, float): self.simple_element("real", repr(value)) - elif isinstance(value, dict): + elif isinstance(value, (dict, frozendict)): self.write_dict(value) elif isinstance(value, (bytes, bytearray)): @@ -715,7 +715,7 @@ def _flatten(self, value): self._objidtable[id(value)] = refnum # And finally recurse into containers - if isinstance(value, dict): + if isinstance(value, (dict, frozendict)): keys = [] values = [] items = value.items() @@ -836,7 +836,7 @@ def _write_object(self, value): self._write_size(0xA0, s) self._fp.write(struct.pack('>' + self._ref_format * s, *refs)) - elif isinstance(value, dict): + elif isinstance(value, (dict, frozendict)): keyRefs, valRefs = [], [] if self._sort_keys: @@ -869,18 +869,18 @@ def _is_fmt_binary(header): # Generic bits # -_FORMATS={ - FMT_XML: dict( +_FORMATS=frozendict({ + FMT_XML: frozendict( detect=_is_fmt_xml, parser=_PlistParser, writer=_PlistWriter, ), - FMT_BINARY: dict( + FMT_BINARY: frozendict( detect=_is_fmt_binary, parser=_BinaryPlistParser, writer=_BinaryPlistWriter, ) -} +}) def load(fp, *, fmt=None, dict_type=dict, aware_datetime=False): diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css index 24e67bedee5242..c4da169d15de88 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css @@ -5,6 +5,46 @@ This file extends the shared foundation with flamegraph-specific styles. ========================================================================== */ +/* -------------------------------------------------------------------------- + Differential Flamegraph + -------------------------------------------------------------------------- */ + +:root { + /* Regression colors */ + --diff-regression-deep: #d32f2f; + --diff-regression-medium: #e57373; + --diff-regression-light: #ef9a9a; + --diff-regression-verylight: #ffcdd2; + + /* Improvement colors */ + --diff-improvement-deep: #1976d2; + --diff-improvement-medium: #42a5f5; + --diff-improvement-light: #64b5f6; + --diff-improvement-verylight: #90caf9; + + /* Other differential colors */ + --diff-neutral: #bdbdbd; + --diff-new: #9575cd; + --diff-elided: #b39ddb; +} + +/* Dark mode differential colors - adjusted for contrast against dark backgrounds */ +[data-theme="dark"] { + --diff-regression-deep: #ef5350; + --diff-regression-medium: #e57373; + --diff-regression-light: #ef9a9a; + --diff-regression-verylight: #ffcdd2; + + --diff-improvement-deep: #42a5f5; + --diff-improvement-medium: #64b5f6; + --diff-improvement-light: #90caf9; + --diff-improvement-verylight: #bbdefb; + + --diff-neutral: #757575; + --diff-new: #b39ddb; + --diff-elided: #ce93d8; +} + /* -------------------------------------------------------------------------- Layout Overrides (Flamegraph-specific) -------------------------------------------------------------------------- */ @@ -277,7 +317,9 @@ body.resizing-sidebar { /* View Mode Section */ .view-mode-section .section-content { display: flex; - justify-content: center; + flex-direction: column; + gap: 10px; + align-items: center; } /* Collapsible sections */ @@ -815,6 +857,41 @@ body.resizing-sidebar { color: var(--accent); } +.tooltip-diff { + margin-top: 12px; + padding-top: 12px; + border-top: 1px solid var(--border); +} + +.tooltip-diff-title { + font-size: 11px; + font-weight: 600; + color: var(--accent); + margin-bottom: 8px; +} + +.tooltip-diff-row { + display: grid; + grid-template-columns: auto 1fr; + gap: 4px 14px; + font-size: 12px; + margin-bottom: 4px; +} + +.tooltip-diff-row.regression .tooltip-stat-value { + color: var(--diff-regression-deep); + font-weight: 700; +} + +.tooltip-diff-row.improvement .tooltip-stat-value { + color: var(--diff-improvement-deep); + font-weight: 700; +} + +.tooltip-diff-row.neutral .tooltip-stat-value { + color: var(--text-secondary); +} + .tooltip-source { margin-top: 10px; padding-top: 10px; @@ -989,7 +1066,8 @@ body.resizing-sidebar { Flamegraph-Specific Toggle Override -------------------------------------------------------------------------- */ -#toggle-invert .toggle-track.on { +#toggle-invert .toggle-track.on, +#toggle-elided .toggle-track.on { background: #8e44ad; border-color: #8e44ad; box-shadow: 0 0 8px rgba(142, 68, 173, 0.3); diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js index a2b21da2970064..d7a8890d4a1ad9 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js @@ -40,44 +40,74 @@ function getOpcodeInfo(opcode) { // String Resolution // ============================================================================ -function resolveString(index) { +function resolveString(index, table = stringTable) { if (index === null || index === undefined) { return null; } - if (typeof index === 'number' && index >= 0 && index < stringTable.length) { - return stringTable[index]; + if (typeof index === 'number' && index >= 0 && index < table.length) { + return table[index]; } return String(index); } -function resolveStringIndices(node) { +function resolveStringIndices(node, table) { if (!node) return node; const resolved = { ...node }; if (typeof resolved.name === 'number') { - resolved.name = resolveString(resolved.name); + resolved.name = resolveString(resolved.name, table); } if (typeof resolved.filename === 'number') { - resolved.filename = resolveString(resolved.filename); + resolved.filename = resolveString(resolved.filename, table); } if (typeof resolved.funcname === 'number') { - resolved.funcname = resolveString(resolved.funcname); + resolved.funcname = resolveString(resolved.funcname, table); } if (Array.isArray(resolved.source)) { resolved.source = resolved.source.map(index => - typeof index === 'number' ? resolveString(index) : index + typeof index === 'number' ? resolveString(index, table) : index ); } if (Array.isArray(resolved.children)) { - resolved.children = resolved.children.map(child => resolveStringIndices(child)); + resolved.children = resolved.children.map(child => resolveStringIndices(child, table)); } return resolved; } +function selectFlamegraphData() { + const baseData = isShowingElided ? elidedFlamegraphData : normalData; + + if (!isInverted) { + return baseData; + } + + if (isShowingElided) { + if (!invertedElidedData) { + invertedElidedData = generateInvertedFlamegraph(baseData); + } + return invertedElidedData; + } + + if (!invertedData) { + invertedData = generateInvertedFlamegraph(baseData); + } + return invertedData; +} + +function updateFlamegraphView() { + const selectedData = selectFlamegraphData(); + const selectedThreadId = currentThreadFilter !== 'all' ? parseInt(currentThreadFilter, 10) : null; + const filteredData = selectedThreadId !== null ? filterDataByThread(selectedData, selectedThreadId) : selectedData; + const tooltip = createPythonTooltip(filteredData); + const chart = createFlamegraph(tooltip, filteredData.value, filteredData); + renderFlamegraph(chart, filteredData); + populateThreadStats(selectedData, selectedThreadId); +} + // ============================================================================ // Theme & UI Controls // ============================================================================ @@ -87,10 +117,7 @@ function toggleTheme() { // Re-render flamegraph with new theme colors if (window.flamegraphData && normalData) { - const currentData = isInverted ? invertedData : normalData; - const tooltip = createPythonTooltip(currentData); - const chart = createFlamegraph(tooltip, currentData.value); - renderFlamegraph(chart, window.flamegraphData); + updateFlamegraphView(); } } @@ -265,6 +292,8 @@ function createPythonTooltip(data) { } const timeMs = (d.data.value / 1000).toFixed(2); + const selfSamples = d.data.self || 0; + const selfMs = (selfSamples / 1000).toFixed(2); const percentage = ((d.data.value / data.value) * 100).toFixed(2); const calls = d.data.calls || 0; const childCount = d.children ? d.children.length : 0; @@ -342,15 +371,48 @@ function createPythonTooltip(data) { const fileLocationHTML = isSpecialFrame ? "" : `
${filename}${d.data.lineno ? ":" + d.data.lineno : ""}
`; + // Differential stats section + let diffSection = ""; + if (d.data.diff !== undefined && d.data.baseline !== undefined) { + const baselineSelf = (d.data.baseline / 1000).toFixed(2); + const currentSelf = ((d.data.self_time || 0) / 1000).toFixed(2); + const diffMs = (d.data.diff / 1000).toFixed(2); + const diffPct = d.data.diff_pct; + const sign = d.data.diff >= 0 ? "+" : ""; + const diffClass = d.data.diff > 0 ? "regression" : (d.data.diff < 0 ? "improvement" : "neutral"); + + diffSection = ` +
+
Self-Time Comparison:
+
+ Baseline Self: + ${baselineSelf} ms +
+
+ Current Self: + ${currentSelf} ms +
+
+ Difference: + ${sign}${diffMs} ms (${sign}${diffPct.toFixed(1)}%) +
+
`; + } + const tooltipHTML = `
${funcname}
${fileLocationHTML}
- Execution Time: + Total Time: ${timeMs} ms + ${selfSamples > 0 ? ` + Self Time: + ${selfMs} ms + ` : ''} + Percentage: ${percentage}% @@ -364,6 +426,7 @@ function createPythonTooltip(data) { ${childCount} ` : ''}
+ ${diffSection} ${sourceSection} ${opcodeSection}
@@ -458,11 +521,64 @@ function getHeatColors() { return colors; } -function createFlamegraph(tooltip, rootValue) { +function getDiffColors() { + const style = getComputedStyle(document.documentElement); + return { + elided: style.getPropertyValue('--diff-elided').trim(), + new: style.getPropertyValue('--diff-new').trim(), + neutral: style.getPropertyValue('--diff-neutral').trim(), + regressionDeep: style.getPropertyValue('--diff-regression-deep').trim(), + regressionMedium: style.getPropertyValue('--diff-regression-medium').trim(), + regressionLight: style.getPropertyValue('--diff-regression-light').trim(), + regressionVerylight: style.getPropertyValue('--diff-regression-verylight').trim(), + improvementDeep: style.getPropertyValue('--diff-improvement-deep').trim(), + improvementMedium: style.getPropertyValue('--diff-improvement-medium').trim(), + improvementLight: style.getPropertyValue('--diff-improvement-light').trim(), + improvementVerylight: style.getPropertyValue('--diff-improvement-verylight').trim(), + }; +} + +function getDiffColorForNode(node, diffColors) { + if (isShowingElided) { + return diffColors.elided; + } + + const diff_pct = node.data.diff_pct || 0; + const diff_samples = node.data.diff || 0; + const self_time = node.data.self_time || 0; + + if (diff_pct === 100 && self_time > 0 && Math.abs(diff_samples - self_time) < 0.1) { + return diffColors.new; + } + + // Neutral zone: small percentage change + if (Math.abs(diff_pct) < 15) { + return diffColors.neutral; + } + + // Regression (red scale) + if (diff_pct > 0) { + if (diff_pct >= 100) return diffColors.regressionDeep; + if (diff_pct > 50) return diffColors.regressionMedium; + if (diff_pct > 30) return diffColors.regressionLight; + return diffColors.regressionVerylight; + } + + // Improvement (blue scale) + if (diff_pct <= -100) return diffColors.improvementDeep; + if (diff_pct < -50) return diffColors.improvementMedium; + if (diff_pct < -30) return diffColors.improvementLight; + return diffColors.improvementVerylight; +} + +function createFlamegraph(tooltip, rootValue, data) { const chartArea = document.querySelector('.chart-area'); const width = chartArea ? chartArea.clientWidth - 32 : window.innerWidth - 320; const heatColors = getHeatColors(); + const isDifferential = data && data.stats && data.stats.is_differential; + const diffColors = isDifferential ? getDiffColors() : null; + let chart = flamegraph() .width(width) .cellHeight(20) @@ -471,9 +587,12 @@ function createFlamegraph(tooltip, rootValue) { .tooltip(tooltip) .inverted(true) .setColorMapper(function (d) { - // Root node should be transparent if (d.depth === 0) return 'transparent'; + if (isDifferential) { + return getDiffColorForNode(d, diffColors); + } + const percentage = d.data.value / rootValue; const level = getHeatLevel(percentage); return heatColors[level]; @@ -857,6 +976,37 @@ function populateProfileSummary(data) { } } +// ============================================================================ +// Elided Stacks (Differential) +// ============================================================================ + +let elidedFlamegraphData = null; +let invertedElidedData = null; +let isShowingElided = false; + +function setupElidedToggle(data) { + const stats = data.stats || {}; + const elidedCount = stats.elided_count || 0; + const elidedFlamegraph = stats.elided_flamegraph; + + if (!elidedCount || !elidedFlamegraph) { + return; + } + + elidedFlamegraphData = resolveStringIndices(elidedFlamegraph, elidedFlamegraph.strings); + + const toggleElided = document.getElementById('toggle-elided'); + if (toggleElided) { + toggleElided.style.display = 'flex'; + + toggleElided.onclick = function() { + isShowingElided = !isShowingElided; + updateToggleUI('toggle-elided', isShowingElided); + updateFlamegraphView(); + }; + } +} + // ============================================================================ // Hotspot Stats // ============================================================================ @@ -868,6 +1018,9 @@ function populateStats(data) { // Populate thread statistics if available populateThreadStats(data); + // Setup elided stacks toggle if this is a differential flamegraph + setupElidedToggle(data); + // For hotspots: use normal (non-inverted) tree structure, but respect thread filtering. // In inverted view, the tree structure changes but the hottest functions remain the same. // However, if a thread filter is active, we need to show that thread's hotspots. @@ -1040,28 +1193,8 @@ function filterByThread() { const selectedThread = threadFilter.value; currentThreadFilter = selectedThread; - const baseData = isInverted ? invertedData : normalData; - - let filteredData; - let selectedThreadId = null; - - if (selectedThread === 'all') { - filteredData = baseData; - } else { - selectedThreadId = parseInt(selectedThread, 10); - filteredData = filterDataByThread(baseData, selectedThreadId); - - if (filteredData.strings) { - stringTable = filteredData.strings; - filteredData = resolveStringIndices(filteredData); - } - } - - const tooltip = createPythonTooltip(filteredData); - const chart = createFlamegraph(tooltip, filteredData.value); - renderFlamegraph(chart, filteredData); - populateThreadStats(baseData, selectedThreadId); + updateFlamegraphView(); } function filterDataByThread(data, threadId) { @@ -1138,54 +1271,89 @@ function getInvertNodeKey(node) { return `${node.filename || '~'}|${node.lineno || 0}|${node.funcname || node.name}`; } -function accumulateInvertedNode(parent, stackFrame, leaf) { +function accumulateInvertedNode(parent, stackFrame, leaf, isDifferential) { const key = getInvertNodeKey(stackFrame); if (!parent.children[key]) { - parent.children[key] = { + const newNode = { name: stackFrame.name, value: 0, + self: 0, children: {}, filename: stackFrame.filename, lineno: stackFrame.lineno, funcname: stackFrame.funcname, source: stackFrame.source, + opcodes: null, threads: new Set() }; + + if (isDifferential) { + newNode.baseline = 0; + newNode.baseline_total = 0; + newNode.self_time = 0; + newNode.diff = 0; + newNode.diff_pct = 0; + } + + parent.children[key] = newNode; } const node = parent.children[key]; node.value += leaf.value; + node.self += stackFrame.self || 0; if (leaf.threads) { leaf.threads.forEach(t => node.threads.add(t)); } + if (stackFrame.opcodes) { + if (!node.opcodes) { + node.opcodes = { ...stackFrame.opcodes }; + } else { + for (const [op, count] of Object.entries(stackFrame.opcodes)) { + node.opcodes[op] = (node.opcodes[op] || 0) + count; + } + } + } + + if (isDifferential) { + node.baseline += stackFrame.baseline || 0; + node.baseline_total += stackFrame.baseline_total || 0; + node.self_time += stackFrame.self_time || 0; + node.diff += stackFrame.diff || 0; + + if (node.baseline > 0) { + node.diff_pct = (node.diff / node.baseline) * 100.0; + } else if (node.self_time > 0) { + node.diff_pct = 100.0; + } + } return node; } -function processLeaf(invertedRoot, path, leafNode) { +function processLeaf(invertedRoot, path, leafNode, isDifferential) { if (!path || path.length === 0) { return; } - let invertedParent = accumulateInvertedNode(invertedRoot, leafNode, leafNode); + let invertedParent = accumulateInvertedNode(invertedRoot, leafNode, leafNode, isDifferential); // Walk backwards through the call stack for (let i = path.length - 2; i >= 0; i--) { - invertedParent = accumulateInvertedNode(invertedParent, path[i], leafNode); + invertedParent = accumulateInvertedNode(invertedParent, path[i], leafNode, isDifferential); } } -function traverseInvert(path, currentNode, invertedRoot) { +function traverseInvert(path, currentNode, invertedRoot, isDifferential) { const children = currentNode.children || []; const childThreads = new Set(children.flatMap(c => c.threads || [])); const selfThreads = (currentNode.threads || []).filter(t => !childThreads.has(t)); if (selfThreads.length > 0) { - processLeaf(invertedRoot, path, { ...currentNode, threads: selfThreads }); + processLeaf(invertedRoot, path, { ...currentNode, threads: selfThreads }, isDifferential); } - children.forEach(child => traverseInvert(path.concat([child]), child, invertedRoot)); + children.forEach(child => traverseInvert(path.concat([child]), child, invertedRoot, isDifferential)); } function convertInvertDictToArray(node) { @@ -1203,6 +1371,8 @@ function convertInvertDictToArray(node) { } function generateInvertedFlamegraph(data) { + const isDifferential = data && data.stats && data.stats.is_differential; + const invertedRoot = { name: data.name, value: data.value, @@ -1214,9 +1384,9 @@ function generateInvertedFlamegraph(data) { const children = data.children || []; if (children.length === 0) { // Single-frame tree: the root is its own leaf - processLeaf(invertedRoot, [data], data); + processLeaf(invertedRoot, [data], data, isDifferential); } else { - children.forEach(child => traverseInvert([child], child, invertedRoot)); + children.forEach(child => traverseInvert([child], child, invertedRoot, isDifferential)); } convertInvertDictToArray(invertedRoot); @@ -1226,21 +1396,7 @@ function generateInvertedFlamegraph(data) { function toggleInvert() { isInverted = !isInverted; updateToggleUI('toggle-invert', isInverted); - - // Build inverted data on first use - if (isInverted && !invertedData) { - invertedData = generateInvertedFlamegraph(normalData); - } - - let dataToRender = isInverted ? invertedData : normalData; - - if (currentThreadFilter !== 'all') { - dataToRender = filterDataByThread(dataToRender, parseInt(currentThreadFilter)); - } - - const tooltip = createPythonTooltip(dataToRender); - const chart = createFlamegraph(tooltip, dataToRender.value); - renderFlamegraph(chart, dataToRender); + updateFlamegraphView(); } // ============================================================================ @@ -1254,7 +1410,7 @@ function initFlamegraph() { if (EMBEDDED_DATA.strings) { stringTable = EMBEDDED_DATA.strings; - normalData = resolveStringIndices(EMBEDDED_DATA); + normalData = resolveStringIndices(EMBEDDED_DATA, EMBEDDED_DATA.strings); } else { normalData = EMBEDDED_DATA; } @@ -1267,8 +1423,20 @@ function initFlamegraph() { initThreadFilter(normalData); + // Toggle legend based on differential mode + const isDifferential = normalData && normalData.stats && normalData.stats.is_differential; + const heatmapLegend = document.getElementById('heatmap-legend-section'); + const diffLegend = document.getElementById('diff-legend-section'); + if (isDifferential) { + if (heatmapLegend) heatmapLegend.style.display = 'none'; + if (diffLegend) diffLegend.style.display = 'block'; + } else { + if (heatmapLegend) heatmapLegend.style.display = 'block'; + if (diffLegend) diffLegend.style.display = 'none'; + } + const tooltip = createPythonTooltip(normalData); - const chart = createFlamegraph(tooltip, normalData.value); + const chart = createFlamegraph(tooltip, normalData.value, normalData); renderFlamegraph(chart, normalData); initSearchHandlers(); initSidebarResize(); diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html b/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html index c0d40b2712beea..9a77178aeff7ec 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html @@ -3,7 +3,7 @@ - Tachyon Profiler - Flamegraph Report + {{TITLE}} @@ -18,7 +18,7 @@ Tachyon - Flamegraph Report + {{SUBTITLE}}
View Mode
+ + +
Flamegraph
@@ -294,9 +301,9 @@

Runtime Stats

- -
+ + +
diff --git a/Lib/profiling/sampling/cli.py b/Lib/profiling/sampling/cli.py index f4b31aad45b922..e22ab158d6a94d 100644 --- a/Lib/profiling/sampling/cli.py +++ b/Lib/profiling/sampling/cli.py @@ -16,7 +16,7 @@ from .errors import SamplingUnknownProcessError, SamplingModuleNotFoundError, SamplingScriptNotFoundError from .sample import sample, sample_live, _is_process_running from .pstats_collector import PstatsCollector -from .stack_collector import CollapsedStackCollector, FlamegraphCollector +from .stack_collector import CollapsedStackCollector, FlamegraphCollector, DiffFlamegraphCollector from .heatmap_collector import HeatmapCollector from .gecko_collector import GeckoCollector from .binary_collector import BinaryCollector @@ -56,6 +56,13 @@ class CustomFormatter( pass +class DiffFlamegraphAction(argparse.Action): + """Custom action for --diff-flamegraph that sets both format and baseline path.""" + def __call__(self, parser, namespace, values, option_string=None): + namespace.format = 'diff_flamegraph' + namespace.diff_baseline = values + + _HELP_DESCRIPTION = """Sample a process's stack frames and generate profiling data. Examples: @@ -85,6 +92,7 @@ class CustomFormatter( "pstats": "pstats", "collapsed": "txt", "flamegraph": "html", + "diff_flamegraph": "html", "gecko": "json", "heatmap": "html", "binary": "bin", @@ -94,6 +102,7 @@ class CustomFormatter( "pstats": PstatsCollector, "collapsed": CollapsedStackCollector, "flamegraph": FlamegraphCollector, + "diff_flamegraph": DiffFlamegraphCollector, "gecko": GeckoCollector, "heatmap": HeatmapCollector, "binary": BinaryCollector, @@ -467,6 +476,12 @@ def _add_format_options(parser, include_compression=True, include_binary=True): dest="format", help="Generate interactive HTML heatmap visualization with line-level sample counts", ) + format_group.add_argument( + "--diff-flamegraph", + metavar="BASELINE", + action=DiffFlamegraphAction, + help="Generate differential flamegraph comparing current profile to BASELINE binary file", + ) if include_binary: format_group.add_argument( "--binary", @@ -475,7 +490,7 @@ def _add_format_options(parser, include_compression=True, include_binary=True): dest="format", help="Generate high-performance binary format (use 'replay' command to convert)", ) - parser.set_defaults(format="pstats") + parser.set_defaults(format="pstats", diff_baseline=None) if include_compression: output_group.add_argument( @@ -545,17 +560,18 @@ def _sort_to_mode(sort_choice): return sort_map.get(sort_choice, SORT_MODE_NSAMPLES) def _create_collector(format_type, sample_interval_usec, skip_idle, opcodes=False, - output_file=None, compression='auto'): + output_file=None, compression='auto', diff_baseline=None): """Create the appropriate collector based on format type. Args: - format_type: The output format ('pstats', 'collapsed', 'flamegraph', 'gecko', 'heatmap', 'binary') + format_type: The output format ('pstats', 'collapsed', 'flamegraph', 'gecko', 'heatmap', 'binary', 'diff_flamegraph') sample_interval_usec: Sampling interval in microseconds skip_idle: Whether to skip idle samples opcodes: Whether to collect opcode information (only used by gecko format for creating interval markers in Firefox Profiler) output_file: Output file path (required for binary format) compression: Compression type for binary format ('auto', 'zstd', 'none') + diff_baseline: Path to baseline binary file for differential flamegraph Returns: A collector instance of the appropriate type @@ -564,6 +580,17 @@ def _create_collector(format_type, sample_interval_usec, skip_idle, opcodes=Fals if collector_class is None: raise ValueError(f"Unknown format: {format_type}") + if format_type == "diff_flamegraph": + if diff_baseline is None: + raise ValueError("Differential flamegraph requires a baseline file") + if not os.path.exists(diff_baseline): + raise ValueError(f"Baseline file not found: {diff_baseline}") + return collector_class( + sample_interval_usec, + baseline_binary_path=diff_baseline, + skip_idle=skip_idle + ) + # Binary format requires output file and compression if format_type == "binary": if output_file is None: @@ -663,7 +690,7 @@ def _handle_output(collector, args, pid, mode): collector.export(filename) # Auto-open browser for HTML output if --browser flag is set - if args.format in ('flamegraph', 'heatmap') and getattr(args, 'browser', False): + if args.format in ('flamegraph', 'diff_flamegraph', 'heatmap') and getattr(args, 'browser', False): _open_in_browser(filename) @@ -756,7 +783,7 @@ def _validate_args(args, parser): ) # Validate --opcodes is only used with compatible formats - opcodes_compatible_formats = ("live", "gecko", "flamegraph", "heatmap", "binary") + opcodes_compatible_formats = ("live", "gecko", "flamegraph", "diff_flamegraph", "heatmap", "binary") if getattr(args, 'opcodes', False) and args.format not in opcodes_compatible_formats: parser.error( f"--opcodes is only compatible with {', '.join('--' + f for f in opcodes_compatible_formats)}." @@ -953,7 +980,8 @@ def _handle_attach(args): collector = _create_collector( args.format, args.sample_interval_usec, skip_idle, args.opcodes, output_file=output_file, - compression=getattr(args, 'compression', 'auto') + compression=getattr(args, 'compression', 'auto'), + diff_baseline=args.diff_baseline ) with _get_child_monitor_context(args, args.pid): @@ -1031,7 +1059,8 @@ def _handle_run(args): collector = _create_collector( args.format, args.sample_interval_usec, skip_idle, args.opcodes, output_file=output_file, - compression=getattr(args, 'compression', 'auto') + compression=getattr(args, 'compression', 'auto'), + diff_baseline=args.diff_baseline ) with _get_child_monitor_context(args, process.pid): @@ -1180,7 +1209,10 @@ def _handle_replay(args): print(f" Sample interval: {interval} us") print(f" Compression: {'zstd' if info.get('compression_type', 0) == 1 else 'none'}") - collector = _create_collector(args.format, interval, skip_idle=False) + collector = _create_collector( + args.format, interval, skip_idle=False, + diff_baseline=args.diff_baseline + ) def progress_callback(current, total): if total > 0: @@ -1206,7 +1238,7 @@ def progress_callback(current, total): collector.export(filename) # Auto-open browser for HTML output if --browser flag is set - if args.format in ('flamegraph', 'heatmap') and getattr(args, 'browser', False): + if args.format in ('flamegraph', 'diff_flamegraph', 'heatmap') and getattr(args, 'browser', False): _open_in_browser(filename) print(f"Replayed {count} samples") diff --git a/Lib/profiling/sampling/constants.py b/Lib/profiling/sampling/constants.py index 58a57700fbdd4a..a364d0b8fde1e0 100644 --- a/Lib/profiling/sampling/constants.py +++ b/Lib/profiling/sampling/constants.py @@ -37,6 +37,7 @@ THREAD_STATUS_UNKNOWN, THREAD_STATUS_GIL_REQUESTED, THREAD_STATUS_HAS_EXCEPTION, + THREAD_STATUS_MAIN_THREAD, ) except ImportError: # Fallback for tests or when module is not available @@ -45,3 +46,4 @@ THREAD_STATUS_UNKNOWN = (1 << 2) THREAD_STATUS_GIL_REQUESTED = (1 << 3) THREAD_STATUS_HAS_EXCEPTION = (1 << 4) + THREAD_STATUS_MAIN_THREAD = (1 << 5) diff --git a/Lib/profiling/sampling/gecko_collector.py b/Lib/profiling/sampling/gecko_collector.py index 28ef9b69bf7968..8986194268b3ce 100644 --- a/Lib/profiling/sampling/gecko_collector.py +++ b/Lib/profiling/sampling/gecko_collector.py @@ -9,7 +9,7 @@ from .collector import Collector, filter_internal_frames from .opcode_utils import get_opcode_info, format_opcode try: - from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU, THREAD_STATUS_UNKNOWN, THREAD_STATUS_GIL_REQUESTED, THREAD_STATUS_HAS_EXCEPTION + from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU, THREAD_STATUS_UNKNOWN, THREAD_STATUS_GIL_REQUESTED, THREAD_STATUS_HAS_EXCEPTION, THREAD_STATUS_MAIN_THREAD except ImportError: # Fallback if module not available (shouldn't happen in normal use) THREAD_STATUS_HAS_GIL = (1 << 0) @@ -17,6 +17,7 @@ THREAD_STATUS_UNKNOWN = (1 << 2) THREAD_STATUS_GIL_REQUESTED = (1 << 3) THREAD_STATUS_HAS_EXCEPTION = (1 << 4) + THREAD_STATUS_MAIN_THREAD = (1 << 5) # Categories matching Firefox Profiler expectations @@ -174,15 +175,16 @@ def collect(self, stack_frames, timestamps_us=None): for thread_info in interpreter_info.threads: frames = filter_internal_frames(thread_info.frame_info) tid = thread_info.thread_id + status_flags = thread_info.status + is_main_thread = bool(status_flags & THREAD_STATUS_MAIN_THREAD) # Initialize thread if needed if tid not in self.threads: - self.threads[tid] = self._create_thread(tid) + self.threads[tid] = self._create_thread(tid, is_main_thread) thread_data = self.threads[tid] # Decode status flags - status_flags = thread_info.status has_gil = bool(status_flags & THREAD_STATUS_HAS_GIL) on_cpu = bool(status_flags & THREAD_STATUS_ON_CPU) gil_requested = bool(status_flags & THREAD_STATUS_GIL_REQUESTED) @@ -288,18 +290,12 @@ def collect(self, stack_frames, timestamps_us=None): self.sample_count += len(times) - def _create_thread(self, tid): + def _create_thread(self, tid, is_main_thread): """Create a new thread structure with processed profile format.""" - # Determine if this is the main thread - try: - is_main = tid == threading.main_thread().ident - except (RuntimeError, AttributeError): - is_main = False - thread = { "name": f"Thread-{tid}", - "isMainThread": is_main, + "isMainThread": is_main_thread, "processStartupTime": 0, "processShutdownTime": None, "registerTime": 0, diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index 931bc2c487b55b..461ce95a25874b 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -19,9 +19,9 @@ def __init__(self, sample_interval_usec, *, skip_idle=False): self.sample_interval_usec = sample_interval_usec self.skip_idle = skip_idle - def collect(self, stack_frames, timestamps_us=None, skip_idle=False): + def collect(self, stack_frames, timestamps_us=None): weight = len(timestamps_us) if timestamps_us else 1 - for frames, thread_id in self._iter_stacks(stack_frames, skip_idle=skip_idle): + for frames, thread_id in self._iter_stacks(stack_frames, skip_idle=self.skip_idle): self.process_frames(frames, thread_id, weight=weight) def process_frames(self, frames, thread_id, weight=1): @@ -88,7 +88,7 @@ def __init__(self, *args, **kwargs): # Per-thread statistics self.per_thread_stats = {} # {thread_id: {has_gil, on_cpu, gil_requested, unknown, has_exception, total, gc_samples}} - def collect(self, stack_frames, timestamps_us=None, skip_idle=False): + def collect(self, stack_frames, timestamps_us=None): """Override to track thread status statistics before processing frames.""" # Weight is number of timestamps (samples with identical stack) weight = len(timestamps_us) if timestamps_us else 1 @@ -123,7 +123,7 @@ def collect(self, stack_frames, timestamps_us=None, skip_idle=False): self.per_thread_stats[thread_id][key] += value * weight # Call parent collect to process frames - super().collect(stack_frames, timestamps_us, skip_idle=skip_idle) + super().collect(stack_frames, timestamps_us) def set_stats(self, sample_interval_usec, duration_sec, sample_rate, error_rate=None, missed_samples=None, mode=None): @@ -207,6 +207,7 @@ def convert_children(children, min_samples): child_entry = { "name": name_idx, "value": samples, + "self": node.get("self", 0), "children": [], "filename": filename_idx, "lineno": func[1], @@ -333,7 +334,7 @@ def process_frames(self, frames, thread_id, weight=1): node = current["children"].get(func) if node is None: - node = {"samples": 0, "children": {}, "threads": set(), "opcodes": collections.Counter()} + node = {"samples": 0, "children": {}, "threads": set(), "opcodes": collections.Counter(), "self": 0} current["children"][func] = node node["samples"] += weight node["threads"].add(thread_id) @@ -343,6 +344,9 @@ def process_frames(self, frames, thread_id, weight=1): current = node + if current is not self._root: + current["self"] += weight + def _get_source_lines(self, func): filename, lineno, _ = func @@ -381,6 +385,18 @@ def _create_flamegraph_html(self, data): component_js = (template_dir / "_flamegraph_assets" / "flamegraph.js").read_text(encoding="utf-8") js_content = f"{base_js}\n{component_js}" + # Set title and subtitle based on whether this is a differential flamegraph + is_differential = data.get("stats", {}).get("is_differential", False) + if is_differential: + title = "Tachyon Profiler - Differential Flamegraph Report" + subtitle = "Differential Flamegraph Report" + else: + title = "Tachyon Profiler - Flamegraph Report" + subtitle = "Flamegraph Report" + + html_template = html_template.replace("{{TITLE}}", title) + html_template = html_template.replace("{{SUBTITLE}}", subtitle) + # Inline first-party CSS/JS html_template = html_template.replace( "", f"" @@ -427,3 +443,266 @@ def _create_flamegraph_html(self, data): ) return html_content + + +class DiffFlamegraphCollector(FlamegraphCollector): + """Differential flamegraph collector that compares against a baseline binary profile.""" + + def __init__(self, sample_interval_usec, *, baseline_binary_path, skip_idle=False): + super().__init__(sample_interval_usec, skip_idle=skip_idle) + if not os.path.exists(baseline_binary_path): + raise ValueError(f"Baseline file not found: {baseline_binary_path}") + self.baseline_binary_path = baseline_binary_path + self._baseline_collector = None + self._elided_paths = set() + + def _load_baseline(self): + """Load baseline profile from binary file.""" + from .binary_reader import BinaryReader + + with BinaryReader(self.baseline_binary_path) as reader: + info = reader.get_info() + + baseline_collector = FlamegraphCollector( + sample_interval_usec=info['sample_interval_us'], + skip_idle=self.skip_idle + ) + + reader.replay_samples(baseline_collector) + + self._baseline_collector = baseline_collector + + def _aggregate_path_samples(self, root_node, path=None): + """Aggregate samples by stack path, excluding line numbers for cross-profile matching.""" + if path is None: + path = () + + stats = {} + + for func, node in root_node["children"].items(): + filename, _lineno, funcname = func + func_key = (filename, funcname) + path_key = path + (func_key,) + + total_samples = node.get("samples", 0) + self_samples = node.get("self", 0) + + if path_key in stats: + stats[path_key]["total"] += total_samples + stats[path_key]["self"] += self_samples + else: + stats[path_key] = { + "total": total_samples, + "self": self_samples + } + + child_stats = self._aggregate_path_samples(node, path_key) + for key, data in child_stats.items(): + if key in stats: + stats[key]["total"] += data["total"] + stats[key]["self"] += data["self"] + else: + stats[key] = data + + return stats + + def _convert_to_flamegraph_format(self): + """Convert to flamegraph format with differential annotations.""" + if self._baseline_collector is None: + self._load_baseline() + + current_flamegraph = super()._convert_to_flamegraph_format() + + current_stats = self._aggregate_path_samples(self._root) + baseline_stats = self._aggregate_path_samples(self._baseline_collector._root) + + # Scale baseline values to make them comparable, accounting for both + # sample count differences and sample interval differences. + baseline_total = self._baseline_collector._total_samples + if baseline_total > 0 and self._total_samples > 0: + current_time = self._total_samples * self.sample_interval_usec + baseline_time = baseline_total * self._baseline_collector.sample_interval_usec + scale = current_time / baseline_time + elif baseline_total > 0: + # Current profile is empty - use interval-based scale for elided display + scale = self.sample_interval_usec / self._baseline_collector.sample_interval_usec + else: + scale = 1.0 + + self._annotate_nodes_with_diff(current_flamegraph, current_stats, baseline_stats, scale) + self._add_elided_flamegraph(current_flamegraph, current_stats, baseline_stats, scale) + + return current_flamegraph + + def _annotate_nodes_with_diff(self, current_flamegraph, current_stats, baseline_stats, scale): + """Annotate each node in the tree with diff metadata.""" + if "stats" not in current_flamegraph: + current_flamegraph["stats"] = {} + + current_flamegraph["stats"]["baseline_samples"] = self._baseline_collector._total_samples + current_flamegraph["stats"]["current_samples"] = self._total_samples + current_flamegraph["stats"]["baseline_scale"] = scale + current_flamegraph["stats"]["is_differential"] = True + + if self._is_promoted_root(current_flamegraph): + self._add_diff_data_to_node(current_flamegraph, (), current_stats, baseline_stats, scale) + else: + for child in current_flamegraph["children"]: + self._add_diff_data_to_node(child, (), current_stats, baseline_stats, scale) + + def _add_diff_data_to_node(self, node, path, current_stats, baseline_stats, scale): + """Recursively add diff metadata to nodes.""" + func_key = self._extract_func_key(node, self._string_table) + path_key = path + (func_key,) if func_key else path + + current_data = current_stats.get(path_key, {"total": 0, "self": 0}) + baseline_data = baseline_stats.get(path_key, {"total": 0, "self": 0}) + + current_self = current_data["self"] + baseline_self = baseline_data["self"] * scale + baseline_total = baseline_data["total"] * scale + + diff = current_self - baseline_self + if baseline_self > 0: + diff_pct = (diff / baseline_self) * 100.0 + elif current_self > 0: + diff_pct = 100.0 + else: + diff_pct = 0.0 + + node["baseline"] = baseline_self + node["baseline_total"] = baseline_total + node["self_time"] = current_self + node["diff"] = diff + node["diff_pct"] = diff_pct + + if "children" in node and node["children"]: + for child in node["children"]: + self._add_diff_data_to_node(child, path_key, current_stats, baseline_stats, scale) + + def _is_promoted_root(self, data): + """Check if the data represents a promoted root node.""" + return "filename" in data and "funcname" in data + + def _add_elided_flamegraph(self, current_flamegraph, current_stats, baseline_stats, scale): + """Calculate elided paths and add elided flamegraph to stats.""" + self._elided_paths = baseline_stats.keys() - current_stats.keys() + + current_flamegraph["stats"]["elided_count"] = len(self._elided_paths) + + if self._elided_paths: + elided_flamegraph = self._build_elided_flamegraph(baseline_stats, scale) + if elided_flamegraph: + current_flamegraph["stats"]["elided_flamegraph"] = elided_flamegraph + + def _build_elided_flamegraph(self, baseline_stats, scale): + """Build flamegraph containing only elided paths from baseline. + + This re-runs the base conversion pipeline on the baseline collector + to produce a complete formatted flamegraph, then prunes it to keep + only elided paths. + """ + if not self._baseline_collector or not self._elided_paths: + return None + + # Suppress source line collection for elided nodes - these functions + # no longer exist in the current profile, so source lines from the + # current machine's filesystem would be misleading or unavailable. + orig_get_source = self._baseline_collector._get_source_lines + self._baseline_collector._get_source_lines = lambda func: None + try: + baseline_data = self._baseline_collector._convert_to_flamegraph_format() + finally: + self._baseline_collector._get_source_lines = orig_get_source + + # Remove non-elided nodes and recalculate values + if not self._extract_elided_nodes(baseline_data, path=()): + return None + + self._add_elided_metadata(baseline_data, baseline_stats, scale, path=()) + + # Merge only profiling metadata, not thread-level stats + for key in ("sample_interval_usec", "duration_sec", "sample_rate", + "error_rate", "missed_samples", "mode"): + if key in self.stats: + baseline_data["stats"][key] = self.stats[key] + baseline_data["stats"]["is_differential"] = True + baseline_data["stats"]["baseline_samples"] = self._baseline_collector._total_samples + baseline_data["stats"]["current_samples"] = self._total_samples + + return baseline_data + + def _extract_elided_nodes(self, node, path): + """Remove non-elided nodes and recalculate values bottom-up.""" + if not node: + return False + + func_key = self._extract_func_key(node, self._baseline_collector._string_table) + current_path = path + (func_key,) if func_key else path + + is_elided = current_path in self._elided_paths if func_key else False + + if "children" in node: + # Filter children, keeping only those with elided descendants + elided_children = [] + total_value = 0 + for child in node["children"]: + if self._extract_elided_nodes(child, current_path): + elided_children.append(child) + total_value += child.get("value", 0) + node["children"] = elided_children + + # Recalculate value for structural (non-elided) ancestor nodes; + # elided nodes keep their original value to preserve self-samples + if elided_children and not is_elided: + node["value"] = total_value + + # Keep this node if it's elided or has elided descendants + return is_elided or bool(node.get("children")) + + def _add_elided_metadata(self, node, baseline_stats, scale, path): + """Add differential metadata showing this path disappeared.""" + if not node: + return + + func_key = self._extract_func_key(node, self._baseline_collector._string_table) + current_path = path + (func_key,) if func_key else path + + if func_key and current_path in baseline_stats: + baseline_data = baseline_stats[current_path] + baseline_self = baseline_data["self"] * scale + baseline_total = baseline_data["total"] * scale + + node["baseline"] = baseline_self + node["baseline_total"] = baseline_total + node["diff"] = -baseline_self + else: + node["baseline"] = 0 + node["baseline_total"] = 0 + node["diff"] = 0 + + node["self_time"] = 0 + # Elided paths have zero current self-time, so the change is always + # -100% when there was actual baseline self-time to lose. + # For internal nodes with no baseline self-time, use 0% to avoid + # misleading tooltips. + if baseline_self > 0: + node["diff_pct"] = -100.0 + else: + node["diff_pct"] = 0.0 + + if "children" in node and node["children"]: + for child in node["children"]: + self._add_elided_metadata(child, baseline_stats, scale, current_path) + + def _extract_func_key(self, node, string_table): + """Extract (filename, funcname) key from node, excluding line numbers. + + Line numbers are excluded to match functions even if they moved. + Returns None for root nodes that don't have function information. + """ + if "filename" not in node or "funcname" not in node: + return None + filename = string_table.get_string(node["filename"]) + funcname = string_table.get_string(node["funcname"]) + return (filename, funcname) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index b943fba3d33f4b..7ac2289f535b6d 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -351,15 +351,16 @@ def _args_from_interpreter_flags(): # -X options if dev_mode: args.extend(('-X', 'dev')) - for opt in ('faulthandler', 'tracemalloc', 'importtime', - 'frozen_modules', 'showrefcount', 'utf8', 'gil'): - if opt in xoptions: - value = xoptions[opt] - if value is True: - arg = opt - else: - arg = '%s=%s' % (opt, value) - args.extend(('-X', arg)) + for opt in sorted(xoptions): + if opt == 'dev': + # handled above via sys.flags.dev_mode + continue + value = xoptions[opt] + if value is True: + arg = opt + else: + arg = '%s=%s' % (opt, value) + args.extend(('-X', arg)) return args diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 6507a7b5b0f695..47415adce04c2c 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -694,11 +694,19 @@ def get_platform(): release = get_config_var("ANDROID_API_LEVEL") # Wheel tags use the ABI names from Android's own tools. + # When Python is running on 32-bit ARM Android on a 64-bit ARM kernel, + # 'os.uname().machine' is 'armv8l'. Such devices run the same userspace + # code as 'armv7l' devices. + # During the build process of the Android testbed when targeting 32-bit ARM, + # '_PYTHON_HOST_PLATFORM' is 'arm-linux-androideabi', so 'machine' becomes + # 'arm'. machine = { - "x86_64": "x86_64", - "i686": "x86", "aarch64": "arm64_v8a", + "arm": "armeabi_v7a", "armv7l": "armeabi_v7a", + "armv8l": "armeabi_v7a", + "i686": "x86", + "x86_64": "x86_64", }[machine] elif osname == "linux": # At least on Linux/Intel, 'machine' is the processor -- diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index e264433ca590bf..bb8695541ac81d 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -48,7 +48,11 @@ try: import _pydatetime except ImportError: - pass + _pydatetime = None +try: + import _datetime +except ImportError: + _datetime = None # pickle_loads = {pickle.loads, pickle._loads} diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 2c404f6d80bcf3..ea26cc849f8182 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -393,6 +393,8 @@ def _create_parser(): help='remove old test_python_* directories') group.add_argument('--bisect', action='store_true', help='if some tests fail, run test.bisect_cmd on them') + group.add_argument('--pythoninfo', action='store_true', + help="run python -m test.pythoninfo before tests") group.add_argument('--dont-add-python-opts', dest='_add_python_opts', action='store_false', help="internal option, don't use it") diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index d8b9605ea49843..ac82541059cc62 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -26,7 +26,7 @@ strip_py_suffix, count, format_duration, printlist, get_temp_dir, get_work_dir, exit_timeout, display_header, cleanup_temp_dir, print_warning, - is_cross_compiled, get_host_runner, + is_cross_compiled, get_host_runner, display_title, EXIT_TIMEOUT) @@ -126,6 +126,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.coverage: bool = ns.trace self.coverage_dir: StrPath | None = ns.coverdir self._tmp_dir: StrPath | None = ns.tempdir + self.pythoninfo: bool = ns.pythoninfo # Randomize self.randomize: bool = ns.randomize @@ -322,9 +323,7 @@ def _run_bisect(self, runtests: RunTests, test: str, progress: str) -> bool: title = f"Bisect {test}" if progress: title = f"{title} ({progress})" - print(title) - print("#" * len(title)) - print() + display_title(title) cmd = runtests.create_python_cmd() cmd.extend([ @@ -345,9 +344,7 @@ def _run_bisect(self, runtests: RunTests, test: str, progress: str) -> bool: exitcode = proc.returncode title = f"{title}: exit code {exitcode}" - print(title) - print("#" * len(title)) - print(flush=True) + display_title(title) if exitcode: print(f"Bisect failed with exit code {exitcode}") @@ -752,6 +749,15 @@ def tmp_dir(self) -> StrPath: ) return self._tmp_dir + def run_pythoninfo(self): + from test import pythoninfo + try: + pythoninfo.main() + except SystemExit: + # Ignore non-zero exit code on purpose + pass + print() + def main(self, tests: TestList | None = None) -> NoReturn: if self.want_add_python_opts: self._add_python_opts() @@ -765,6 +771,9 @@ def main(self, tests: TestList | None = None) -> NoReturn: if self.want_wait: input("Press any key to continue...") + if self.pythoninfo: + self.run_pythoninfo() + setup_test_dir(self.test_dir) selected, tests = self.find_tests(tests) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 7cc9d0bf262af1..00703d6c074855 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -452,12 +452,6 @@ def get_temp_dir(tmp_dir: StrPath | None = None) -> StrPath: f"unexpectedly returned {tmp_dir!r} on WASI" ) tmp_dir = os.path.join(tmp_dir, 'build') - - # When get_temp_dir() is called in a worker process, - # get_temp_dir() path is different than in the parent process - # which is not a WASI process. So the parent does not create - # the same "tmp_dir" than the test worker process. - os.makedirs(tmp_dir, exist_ok=True) else: tmp_dir = tempfile.gettempdir() @@ -752,3 +746,9 @@ def _sanitize_xml_replace(regs): def sanitize_xml(text: str) -> str: return ILLEGAL_XML_CHARS_RE.sub(_sanitize_xml_replace, text) + + +def display_title(title): + print(title) + print("#" * len(title)) + print(flush=True) diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 1ad67e1cebf288..4e69ab9d8fad05 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -127,6 +127,9 @@ def main() -> NoReturn: worker_json = sys.argv[1] tmp_dir = get_temp_dir() + # get_temp_dir() can be different in the worker and the parent process. + # For example, if --tempdir option is used. + os.makedirs(tmp_dir, exist_ok=True) work_dir = get_work_dir(tmp_dir, worker=True) with exit_timeout(): diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 6ac4b19da3ea9c..6366f12257f3b5 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -57,6 +57,8 @@ # kind of outer loop. protocols = range(pickle.HIGHEST_PROTOCOL + 1) +FAST_NESTING_LIMIT = 50 + # Return True if opcode code appears in the pickle, else False. def opcode_in_pickle(code, pickle): @@ -4552,6 +4554,94 @@ def __reduce__(self): expected = "changed size during iteration" self.assertIn(expected, str(e)) + def fast_save_enter(self, create_data, minprotocol=0): + # gh-146059: Check that fast_save_leave() is called when + # fast_save_enter() is called. + if not hasattr(self, "pickler"): + self.skipTest("need Pickler class") + + data = [create_data(i) for i in range(FAST_NESTING_LIMIT * 2)] + protocols = range(minprotocol, pickle.HIGHEST_PROTOCOL + 1) + for proto in protocols: + with self.subTest(proto=proto): + buf = io.BytesIO() + pickler = self.pickler(buf, protocol=proto) + # Enable fast mode (disables memo, enables cycle detection) + pickler.fast = 1 + pickler.dump(data) + + buf.seek(0) + data2 = self.unpickler(buf).load() + self.assertEqual(data2, data) + + def test_fast_save_enter_tuple(self): + self.fast_save_enter(lambda i: (i,)) + + def test_fast_save_enter_list(self): + self.fast_save_enter(lambda i: [i]) + + def test_fast_save_enter_frozenset(self): + self.fast_save_enter(lambda i: frozenset([i])) + + def test_fast_save_enter_set(self): + self.fast_save_enter(lambda i: set([i])) + + def test_fast_save_enter_frozendict(self): + if self.py_version < (3, 15): + self.skipTest('need frozendict') + self.fast_save_enter(lambda i: frozendict(key=i), minprotocol=2) + + def test_fast_save_enter_dict(self): + self.fast_save_enter(lambda i: {"key": i}) + + def deep_nested_struct(self, create_nested, + minprotocol=0, compare_equal=True, + depth=FAST_NESTING_LIMIT * 2): + # gh-146059: Check that fast_save_leave() is called when + # fast_save_enter() is called. + if not hasattr(self, "pickler"): + self.skipTest("need Pickler class") + + data = None + for i in range(depth): + data = create_nested(data) + protocols = range(minprotocol, pickle.HIGHEST_PROTOCOL + 1) + for proto in protocols: + with self.subTest(proto=proto): + buf = io.BytesIO() + pickler = self.pickler(buf, protocol=proto) + # Enable fast mode (disables memo, enables cycle detection) + pickler.fast = 1 + pickler.dump(data) + + buf.seek(0) + data2 = self.unpickler(buf).load() + if compare_equal: + self.assertEqual(data2, data) + + def test_deep_nested_struct_tuple(self): + self.deep_nested_struct(lambda data: (data,)) + + def test_deep_nested_struct_list(self): + self.deep_nested_struct(lambda data: [data]) + + def test_deep_nested_struct_frozenset(self): + self.deep_nested_struct(lambda data: frozenset((1, data))) + + def test_deep_nested_struct_set(self): + self.deep_nested_struct(lambda data: {K(data)}, + depth=FAST_NESTING_LIMIT+1, + compare_equal=False) + + def test_deep_nested_struct_frozendict(self): + if self.py_version < (3, 15): + self.skipTest('need frozendict') + self.deep_nested_struct(lambda data: frozendict(x=data), + minprotocol=2) + + def test_deep_nested_struct_dict(self): + self.deep_nested_struct(lambda data: {'x': data}) + class BigmemPickleTests: diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 219fbb4bb1bbe2..7f735d75b318e7 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -1103,9 +1103,9 @@ def collect_info(info): def dump_info(info, file=None): - title = "Python debug information" + title = "Python build information" print(title) - print("=" * len(title)) + print("#" * len(title)) print() infos = info.get_infos() diff --git a/Lib/test/seq_tests.py b/Lib/test/seq_tests.py index a41970d8f3f55a..b7875fe8f2f75d 100644 --- a/Lib/test/seq_tests.py +++ b/Lib/test/seq_tests.py @@ -261,23 +261,20 @@ def test_minmax(self): self.assertEqual(min(u), 0) self.assertEqual(max(u), 2) - def test_addmul(self): + def test_add(self): u1 = self.type2test([0]) u2 = self.type2test([0, 1]) self.assertEqual(u1, u1 + self.type2test()) self.assertEqual(u1, self.type2test() + u1) self.assertEqual(u1 + self.type2test([1]), u2) self.assertEqual(self.type2test([-1]) + u1, self.type2test([-1, 0])) - self.assertEqual(self.type2test(), u2*0) - self.assertEqual(self.type2test(), 0*u2) + + def test_mul(self): + u2 = self.type2test([0, 1]) self.assertEqual(self.type2test(), u2*0) self.assertEqual(self.type2test(), 0*u2) self.assertEqual(u2, u2*1) self.assertEqual(u2, 1*u2) - self.assertEqual(u2, u2*1) - self.assertEqual(u2, 1*u2) - self.assertEqual(u2+u2, u2*2) - self.assertEqual(u2+u2, 2*u2) self.assertEqual(u2+u2, u2*2) self.assertEqual(u2+u2, 2*u2) self.assertEqual(u2+u2+u2, u2*3) @@ -286,8 +283,9 @@ def test_addmul(self): class subclass(self.type2test): pass u3 = subclass([0, 1]) - self.assertEqual(u3, u3*1) - self.assertIsNot(u3, u3*1) + r = u3*1 + self.assertEqual(r, u3) + self.assertIsNot(r, u3) def test_iadd(self): u = self.type2test([0, 1]) @@ -348,6 +346,21 @@ def test_subscript(self): self.assertRaises(ValueError, a.__getitem__, slice(0, 10, 0)) self.assertRaises(TypeError, a.__getitem__, 'x') + def _assert_cmp(self, a, b, r): + self.assertIs(a == b, r == 0) + self.assertIs(a != b, r != 0) + self.assertIs(a > b, r > 0) + self.assertIs(a <= b, r <= 0) + self.assertIs(a < b, r < 0) + self.assertIs(a >= b, r >= 0) + + def test_cmp(self): + a = self.type2test([0, 1]) + self._assert_cmp(a, a, 0) + self._assert_cmp(a, self.type2test([0, 1]), 0) + self._assert_cmp(a, self.type2test([0]), 1) + self._assert_cmp(a, self.type2test([0, 2]), -1) + def test_count(self): a = self.type2test([0, 1, 2])*3 self.assertEqual(a.count(0), 3) diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index 185aa3fce39149..f66051531faaa1 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -102,6 +102,43 @@ def _get_teststrings(self, charset, digits): teststrings = [self.fixtype(ts) for ts in teststrings] return teststrings + def test_add(self): + s = self.fixtype('ab') + self.assertEqual(s + self.fixtype(''), s) + self.assertEqual(self.fixtype('') + s, s) + self.assertEqual(s + self.fixtype('cd'), self.fixtype('abcd')) + + def test_mul(self): + s = self.fixtype('ab') + self.assertEqual(s*0, self.fixtype('')) + self.assertEqual(0*s, self.fixtype('')) + self.assertEqual(s*1, s) + self.assertEqual(1*s, s) + self.assertEqual(s*2, self.fixtype('abab')) + self.assertEqual(2*s, self.fixtype('abab')) + + class subclass(self.type2test): + pass + s = subclass(self.fixtype('ab')) + r = s*1 + self.assertEqual(r, s) + self.assertIsNot(r, s) + + def _assert_cmp(self, a, b, r): + self.assertIs(a == b, r == 0) + self.assertIs(a != b, r != 0) + self.assertIs(a > b, r > 0) + self.assertIs(a <= b, r <= 0) + self.assertIs(a < b, r < 0) + self.assertIs(a >= b, r >= 0) + + def test_cmp(self): + a = self.fixtype('ab') + self._assert_cmp(a, a, 0) + self._assert_cmp(a, self.fixtype('ab'), 0) + self._assert_cmp(a, self.fixtype('a'), 1) + self._assert_cmp(a, self.fixtype('ac'), -1) + def test_count(self): self.checkequal(3, 'aaa', 'count', 'a') self.checkequal(0, 'aaa', 'count', 'b') @@ -1304,6 +1341,7 @@ def test_extended_getslice(self): slice(start, stop, step)) def test_mul(self): + super().test_mul() self.checkequal('', 'abc', '__mul__', -1) self.checkequal('', 'abc', '__mul__', 0) self.checkequal('abc', 'abc', '__mul__', 1) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 3da662b0c4d50a..8ff061e074074f 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -71,7 +71,8 @@ "BrokenIter", "in_systemd_nspawn_sync_suppressed", "run_no_yield_async_fn", "run_yielding_async_fn", "async_yield", - "reset_code", "on_github_actions" + "reset_code", "on_github_actions", + "requires_root_user", "requires_non_root_user", ] @@ -3317,3 +3318,8 @@ def control_characters_c0() -> list[str]: C0 control characters defined as the byte range 0x00-0x1F, and 0x7F. """ return [chr(c) for c in range(0x00, 0x20)] + ["\x7F"] + + +_ROOT_IN_POSIX = hasattr(os, 'geteuid') and os.geteuid() == 0 +requires_root_user = unittest.skipUnless(_ROOT_IN_POSIX, "test needs root privilege") +requires_non_root_user = unittest.skipIf(_ROOT_IN_POSIX, "test needs non-root account") diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index e89d6c0b1613ba..50cf8fcb6b4ed6 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -646,6 +646,31 @@ def foo(): get_annotations(foo, format=Format.FORWARDREF, eval_str=True) get_annotations(foo, format=Format.STRING, eval_str=True) + def test_eval_str_wrapped_cycle_self(self): + # gh-146556: self-referential __wrapped__ cycle must not hang. + def f(x: 'int') -> 'str': ... + f.__wrapped__ = f + # Cycle is detected and broken; globals from f itself are used. + result = get_annotations(f, eval_str=True) + self.assertEqual(result, {'x': int, 'return': str}) + + def test_eval_str_wrapped_cycle_mutual(self): + # gh-146556: mutual __wrapped__ cycle (a -> b -> a) must not hang. + def a(x: 'int'): ... + def b(): ... + a.__wrapped__ = b + b.__wrapped__ = a + result = get_annotations(a, eval_str=True) + self.assertEqual(result, {'x': int}) + + def test_eval_str_wrapped_chain_no_cycle(self): + # gh-146556: a valid (non-cyclic) __wrapped__ chain must still work. + def inner(x: 'int'): ... + def outer(x: 'int'): ... + outer.__wrapped__ = inner + result = get_annotations(outer, eval_str=True) + self.assertEqual(result, {'x': int}) + def test_stock_annotations(self): def foo(a: int, b: str): pass diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 5c919aea24ed94..4493349798d9c8 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -31,7 +31,7 @@ class ArraySubclassWithKwargs(array.array): def __init__(self, typecode, newarg=None): array.array.__init__(self) -typecodes = 'uwbBhHiIlLfdqQ' +typecodes = 'uwbBhHiIlLfdqQFDe' class MiscTest(unittest.TestCase): @@ -113,6 +113,14 @@ def __index__(self): UTF16_BE = 19 UTF32_LE = 20 UTF32_BE = 21 +IEEE_754_FLOAT_COMPLEX_LE = 22 +IEEE_754_FLOAT_COMPLEX_BE = 23 +IEEE_754_DOUBLE_COMPLEX_LE = 24 +IEEE_754_DOUBLE_COMPLEX_BE = 25 +IEEE_754_FLOAT16_LE = 26 +IEEE_754_FLOAT16_BE = 27 + +MACHINE_FORMAT_CODE_MAX = 27 class ArrayReconstructorTest(unittest.TestCase): @@ -139,7 +147,7 @@ def test_error(self): self.assertRaises(ValueError, array_reconstructor, array.array, "b", UNKNOWN_FORMAT, b"") self.assertRaises(ValueError, array_reconstructor, - array.array, "b", 22, b"") + array.array, "b", MACHINE_FORMAT_CODE_MAX + 1, b"") self.assertRaises(ValueError, array_reconstructor, array.array, "d", 16, b"a") @@ -191,7 +199,15 @@ def test_numbers(self): (['d'], IEEE_754_DOUBLE_LE, 'dddd', - [9006104071832581.0, float('inf'), float('-inf'), -0.0]) + [9006104071832581.0, float('inf'), float('-inf'), -0.0]), + (['F'], IEEE_754_FLOAT_COMPLEX_LE, 'FFFF', + [16711938.0j, float('inf'), complex('1-infj'), -0.0]), + (['D'], IEEE_754_DOUBLE_COMPLEX_LE, 'DDDD', + [9006104071832581.0j, float('inf'), complex('1-infj'), -0.0]), ) for testcase in testcases: valid_typecodes, mformat_code, struct_fmt, values = testcase @@ -279,7 +295,7 @@ def test_byteswap(self): example = self.example a = array.array(self.typecode, example) self.assertRaises(TypeError, a.byteswap, 42) - if a.itemsize in (1, 2, 4, 8): + if a.itemsize in (1, 2, 4, 8, 16): b = array.array(self.typecode, example) b.byteswap() if a.itemsize==1: @@ -1525,6 +1541,62 @@ def test_byteswap(self): b.byteswap() self.assertEqual(a, b) +class CFPTest(NumberTest): + example = [-42j, 0, 42+1j, 1e5j, -1e10] + outside = 23 + + def assertEntryEqual(self, entry1, entry2): + self.assertAlmostEqual(entry1, entry2) + + def test_cmp(self): + a = array.array(self.typecode, self.example) + self.assertIs(a == 42, False) + self.assertIs(a != 42, True) + + self.assertIs(a == a, True) + self.assertIs(a != a, False) + self.assertIs(a < a, False) + self.assertIs(a <= a, True) + self.assertIs(a > a, False) + self.assertIs(a >= a, True) + + self.assertIs(a == 2*a, False) + self.assertIs(a != 2*a, True) + self.assertIs(a < 2*a, True) + self.assertIs(a <= 2*a, True) + self.assertIs(a > 2*a, False) + self.assertIs(a >= 2*a, False) + + def test_nan(self): + a = array.array(self.typecode, [float('nan')]) + b = array.array(self.typecode, [float('nan')]) + self.assertIs(a != b, True) + self.assertIs(a == b, False) + + def test_byteswap(self): + a = array.array(self.typecode, self.example) + self.assertRaises(TypeError, a.byteswap, 42) + if a.itemsize in (1, 2, 4, 8): + b = array.array(self.typecode, self.example) + b.byteswap() + if a.itemsize == 1: + self.assertEqual(a, b) + else: + # On alphas treating the byte swapped bit patterns as + # floats/doubles results in floating-point exceptions + # => compare the 8bit string values instead + self.assertNotEqual(a.tobytes(), b.tobytes()) + b.byteswap() + self.assertEqual(a, b) + + +class HalfFloatTest(FPTest, unittest.TestCase): + example = [-42.0, 0, 42, 1e2, -1e4] + smallerexample = [-42.0, 0, 42, 1e2, -2e4] + biggerexample = [-42.0, 0, 42, 1e2, 1e4] + typecode = 'e' + minitemsize = 2 + class FloatTest(FPTest, unittest.TestCase): typecode = 'f' minitemsize = 4 @@ -1551,6 +1623,15 @@ def test_alloc_overflow(self): self.fail("Array of size > maxsize created - MemoryError expected") +class ComplexFloatTest(CFPTest, unittest.TestCase): + typecode = 'F' + minitemsize = 8 + +class ComplexDoubleTest(CFPTest, unittest.TestCase): + typecode = 'D' + minitemsize = 16 + + class LargeArrayTest(unittest.TestCase): typecode = 'b' diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 8c02de77c24740..e59bc25668b4cb 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -1696,7 +1696,8 @@ def mock_make_ssl_transport(sock, protocol, sslcontext, waiter, server_side=False, server_hostname='python.org', ssl_handshake_timeout=handshake_timeout, - ssl_shutdown_timeout=shutdown_timeout) + ssl_shutdown_timeout=shutdown_timeout, + context=ANY) # Next try an explicit server_hostname. self.loop._make_ssl_transport.reset_mock() coro = self.loop.create_connection( @@ -1711,7 +1712,8 @@ def mock_make_ssl_transport(sock, protocol, sslcontext, waiter, server_side=False, server_hostname='perl.com', ssl_handshake_timeout=handshake_timeout, - ssl_shutdown_timeout=shutdown_timeout) + ssl_shutdown_timeout=shutdown_timeout, + context=ANY) # Finally try an explicit empty server_hostname. self.loop._make_ssl_transport.reset_mock() coro = self.loop.create_connection( @@ -1726,7 +1728,8 @@ def mock_make_ssl_transport(sock, protocol, sslcontext, waiter, server_side=False, server_hostname='', ssl_handshake_timeout=handshake_timeout, - ssl_shutdown_timeout=shutdown_timeout) + ssl_shutdown_timeout=shutdown_timeout, + context=ANY) def test_create_connection_no_ssl_server_hostname_errors(self): # When not using ssl, server_hostname must be None. @@ -2104,7 +2107,7 @@ def test_accept_connection_exception(self, m_log): constants.ACCEPT_RETRY_DELAY, # self.loop._start_serving mock.ANY, - MyProto, sock, None, None, mock.ANY, mock.ANY, mock.ANY) + MyProto, sock, None, None, mock.ANY, mock.ANY, mock.ANY, mock.ANY) def test_call_coroutine(self): async def simple_coroutine(): diff --git a/Lib/test/test_asyncio/test_server_context.py b/Lib/test/test_asyncio/test_server_context.py new file mode 100644 index 00000000000000..3f15654a1af467 --- /dev/null +++ b/Lib/test/test_asyncio/test_server_context.py @@ -0,0 +1,314 @@ +import asyncio +import contextvars +import unittest +import sys + +from unittest import TestCase + +try: + import ssl +except ImportError: + ssl = None + +from test.test_asyncio import utils as test_utils + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + +class ServerContextvarsTestCase: + loop_factory = None # To be defined in subclasses + server_ssl_context = None # To be defined in subclasses for SSL tests + client_ssl_context = None # To be defined in subclasses for SSL tests + + def run_coro(self, coro): + return asyncio.run(coro, loop_factory=self.loop_factory) + + def test_start_server1(self): + # Test that asyncio.start_server captures the context at the time of server creation + async def test(): + var = contextvars.ContextVar("var", default="default") + + async def handle_client(reader, writer): + value = var.get() + writer.write(value.encode()) + await writer.drain() + writer.close() + + server = await asyncio.start_server(handle_client, '127.0.0.1', 0, + ssl=self.server_ssl_context) + # change the value + var.set("after_server") + + async def client(addr): + reader, writer = await asyncio.open_connection(*addr, + ssl=self.client_ssl_context) + data = await reader.read(100) + writer.close() + await writer.wait_closed() + return data.decode() + + async with server: + addr = server.sockets[0].getsockname() + self.assertEqual(await client(addr), "default") + + self.assertEqual(var.get(), "after_server") + + self.run_coro(test()) + + def test_start_server2(self): + # Test that mutations to the context in one handler don't affect other handlers or the server's context + async def test(): + var = contextvars.ContextVar("var", default="default") + + async def handle_client(reader, writer): + value = var.get() + writer.write(value.encode()) + var.set("in_handler") + await writer.drain() + writer.close() + + server = await asyncio.start_server(handle_client, '127.0.0.1', 0, + ssl=self.server_ssl_context) + var.set("after_server") + + async def client(addr): + reader, writer = await asyncio.open_connection(*addr, + ssl=self.client_ssl_context) + data = await reader.read(100) + writer.close() + await writer.wait_closed() + return data.decode() + + async with server: + addr = server.sockets[0].getsockname() + self.assertEqual(await client(addr), "default") + self.assertEqual(await client(addr), "default") + self.assertEqual(await client(addr), "default") + + self.assertEqual(var.get(), "after_server") + + self.run_coro(test()) + + def test_start_server3(self): + # Test that mutations to context in concurrent handlers don't affect each other or the server's context + async def test(): + var = contextvars.ContextVar("var", default="default") + var.set("before_server") + + async def handle_client(reader, writer): + writer.write(var.get().encode()) + await writer.drain() + writer.close() + + server = await asyncio.start_server(handle_client, '127.0.0.1', 0, + ssl=self.server_ssl_context) + var.set("after_server") + + async def client(addr): + reader, writer = await asyncio.open_connection(*addr, + ssl=self.client_ssl_context) + data = await reader.read(100) + self.assertEqual(data.decode(), "before_server") + writer.close() + await writer.wait_closed() + + async with server: + addr = server.sockets[0].getsockname() + async with asyncio.TaskGroup() as tg: + for _ in range(100): + tg.create_task(client(addr)) + + self.assertEqual(var.get(), "after_server") + + self.run_coro(test()) + + def test_create_server1(self): + # Test that loop.create_server captures the context at the time of server creation + # and that mutations to the context in protocol callbacks don't affect the server's context + async def test(): + var = contextvars.ContextVar("var", default="default") + + class EchoProtocol(asyncio.Protocol): + def connection_made(self, transport): + self.transport = transport + value = var.get() + var.set("in_handler") + self.transport.write(value.encode()) + self.transport.close() + + server = await asyncio.get_running_loop().create_server( + lambda: EchoProtocol(), '127.0.0.1', 0, + ssl=self.server_ssl_context) + var.set("after_server") + + async def client(addr): + reader, writer = await asyncio.open_connection(*addr, + ssl=self.client_ssl_context) + data = await reader.read(100) + self.assertEqual(data.decode(), "default") + writer.close() + await writer.wait_closed() + + async with server: + addr = server.sockets[0].getsockname() + await client(addr) + + self.assertEqual(var.get(), "after_server") + + self.run_coro(test()) + + def test_create_server2(self): + # Test that mutations to context in one protocol instance don't affect other instances or the server's context + async def test(): + var = contextvars.ContextVar("var", default="default") + + class EchoProtocol(asyncio.Protocol): + def __init__(self): + super().__init__() + assert var.get() == "default", var.get() + def connection_made(self, transport): + self.transport = transport + value = var.get() + var.set("in_handler") + self.transport.write(value.encode()) + self.transport.close() + + server = await asyncio.get_running_loop().create_server( + lambda: EchoProtocol(), '127.0.0.1', 0, + ssl=self.server_ssl_context) + + var.set("after_server") + + async def client(addr, expected): + reader, writer = await asyncio.open_connection(*addr, + ssl=self.client_ssl_context) + data = await reader.read(100) + self.assertEqual(data.decode(), expected) + writer.close() + await writer.wait_closed() + + async with server: + addr = server.sockets[0].getsockname() + await client(addr, "default") + await client(addr, "default") + + self.assertEqual(var.get(), "after_server") + + self.run_coro(test()) + + def test_gh140947(self): + # See https://github.com/python/cpython/issues/140947 + + cvar1 = contextvars.ContextVar("cvar1") + cvar2 = contextvars.ContextVar("cvar2") + cvar3 = contextvars.ContextVar("cvar3") + results = {} + is_ssl = self.server_ssl_context is not None + + def capture_context(meth): + result = [] + for k,v in contextvars.copy_context().items(): + if k.name.startswith("cvar"): + result.append((k.name, v)) + results[meth] = sorted(result) + + class DemoProtocol(asyncio.Protocol): + def __init__(self, on_conn_lost): + self.transport = None + self.on_conn_lost = on_conn_lost + self.tasks = set() + + def connection_made(self, transport): + capture_context("connection_made") + self.transport = transport + + def data_received(self, data): + capture_context("data_received") + + task = asyncio.create_task(self.asgi()) + self.tasks.add(task) + task.add_done_callback(self.tasks.discard) + + self.transport.pause_reading() + + def connection_lost(self, exc): + capture_context("connection_lost") + if not self.on_conn_lost.done(): + self.on_conn_lost.set_result(True) + + async def asgi(self): + capture_context("asgi start") + cvar1.set(True) + # make sure that we only resume after the pause + # otherwise the resume does nothing + if is_ssl: + while not self.transport._ssl_protocol._app_reading_paused: + await asyncio.sleep(0.01) + else: + while not self.transport._paused: + await asyncio.sleep(0.01) + cvar2.set(True) + self.transport.resume_reading() + cvar3.set(True) + capture_context("asgi end") + + async def main(): + loop = asyncio.get_running_loop() + on_conn_lost = loop.create_future() + + server = await loop.create_server( + lambda: DemoProtocol(on_conn_lost), '127.0.0.1', 0, + ssl=self.server_ssl_context) + async with server: + addr = server.sockets[0].getsockname() + reader, writer = await asyncio.open_connection(*addr, + ssl=self.client_ssl_context) + writer.write(b"anything") + await writer.drain() + writer.close() + await writer.wait_closed() + await on_conn_lost + + self.run_coro(main()) + self.assertDictEqual(results, { + "connection_made": [], + "data_received": [], + "asgi start": [], + "asgi end": [("cvar1", True), ("cvar2", True), ("cvar3", True)], + "connection_lost": [], + }) + + +class AsyncioEventLoopTests(TestCase, ServerContextvarsTestCase): + loop_factory = staticmethod(asyncio.new_event_loop) + +@unittest.skipUnless(ssl, "SSL not available") +class AsyncioEventLoopSSLTests(AsyncioEventLoopTests): + def setUp(self): + super().setUp() + self.server_ssl_context = test_utils.simple_server_sslcontext() + self.client_ssl_context = test_utils.simple_client_sslcontext() + +if sys.platform == "win32": + class AsyncioProactorEventLoopTests(TestCase, ServerContextvarsTestCase): + loop_factory = asyncio.ProactorEventLoop + + class AsyncioSelectorEventLoopTests(TestCase, ServerContextvarsTestCase): + loop_factory = asyncio.SelectorEventLoop + + @unittest.skipUnless(ssl, "SSL not available") + class AsyncioProactorEventLoopSSLTests(AsyncioProactorEventLoopTests): + def setUp(self): + super().setUp() + self.server_ssl_context = test_utils.simple_server_sslcontext() + self.client_ssl_context = test_utils.simple_client_sslcontext() + + @unittest.skipUnless(ssl, "SSL not available") + class AsyncioSelectorEventLoopSSLTests(AsyncioSelectorEventLoopTests): + def setUp(self): + super().setUp() + self.server_ssl_context = test_utils.simple_server_sslcontext() + self.client_ssl_context = test_utils.simple_client_sslcontext() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index a480e16e81bb91..62cfcf8ceb5f2a 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -388,8 +388,8 @@ def close(self): else: # pragma: no cover raise AssertionError("Time generator is not finished") - def _add_reader(self, fd, callback, *args): - self.readers[fd] = events.Handle(callback, args, self, None) + def _add_reader(self, fd, callback, *args, context=None): + self.readers[fd] = events.Handle(callback, args, self, context) def _remove_reader(self, fd): self.remove_reader_count[fd] += 1 @@ -414,8 +414,8 @@ def assert_no_reader(self, fd): if fd in self.readers: raise AssertionError(f'fd {fd} is registered') - def _add_writer(self, fd, callback, *args): - self.writers[fd] = events.Handle(callback, args, self, None) + def _add_writer(self, fd, callback, *args, context=None): + self.writers[fd] = events.Handle(callback, args, self, context) def _remove_writer(self, fd): self.remove_writer_count[fd] += 1 diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index 9648624b267a54..1a4dd56a553f4d 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -209,30 +209,54 @@ def test_b64encode(self): b'\xd3V\xbeo\xf7\x1d', b'01a-b_cd') self.check_encode_type_errors(base64.urlsafe_b64encode) - def test_b64encode_wrapcol(self): + def test_b64encode_padded(self): + b64encode = base64.b64encode + self.assertEqual(b64encode(b'', padded=False), b'') + self.assertEqual(b64encode(b'a', padded=False), b'YQ') + self.assertEqual(b64encode(b'ab', padded=False), b'YWI') + self.assertEqual(b64encode(b'abc', padded=False), b'YWJj') + self.assertEqual(b64encode(b'\xfb', padded=False, altchars=b'-_'), b'-w') + self.assertEqual(b64encode(b'\xfb\xff', padded=False, altchars=b'-_'), + b'-_8') + self.assertEqual(b64encode(b'\xfb\xff\xbf', padded=False, altchars=b'-_'), + b'-_-_') + + urlsafe_b64encode = base64.urlsafe_b64encode + self.assertEqual(urlsafe_b64encode(b'', padded=False), b'') + self.assertEqual(urlsafe_b64encode(b'\xfb', padded=False), b'-w') + self.assertEqual(urlsafe_b64encode(b'\xfb\xff', padded=False), b'-_8') + self.assertEqual(urlsafe_b64encode(b'\xfb\xff\xbf', padded=False), + b'-_-_') + + def _common_test_wrapcol(self, func, data): eq = self.assertEqual - b = b'www.python.org' - eq(base64.b64encode(b, wrapcol=0), b'd3d3LnB5dGhvbi5vcmc=') - eq(base64.b64encode(b, wrapcol=8), b'd3d3LnB5\ndGhvbi5v\ncmc=') - eq(base64.b64encode(b, wrapcol=11), b'd3d3LnB5\ndGhvbi5v\ncmc=') - eq(base64.b64encode(b, wrapcol=76), b'd3d3LnB5dGhvbi5vcmc=') - eq(base64.b64encode(b, wrapcol=1), b'd3d3\nLnB5\ndGhv\nbi5v\ncmc=') - eq(base64.b64encode(b, wrapcol=sys.maxsize), b'd3d3LnB5dGhvbi5vcmc=') + expected = func(data) + eq(func(data, wrapcol=0), expected) + eq(func(data, wrapcol=80), expected) + eq(func(b'', wrapcol=0), func(b'')) + eq(func(b'', wrapcol=1), func(b'')) + eq(func(data, wrapcol=sys.maxsize), expected) if check_impl_detail(): - eq(base64.b64encode(b, wrapcol=sys.maxsize*2), - b'd3d3LnB5dGhvbi5vcmc=') + eq(func(data, wrapcol=sys.maxsize*2), expected) with self.assertRaises(OverflowError): - base64.b64encode(b, wrapcol=2**1000) + func(data, wrapcol=2**1000) with self.assertRaises(ValueError): - base64.b64encode(b, wrapcol=-8) + func(data, wrapcol=-80) with self.assertRaises(TypeError): - base64.b64encode(b, wrapcol=8.0) + func(data, wrapcol=80.0) with self.assertRaises(TypeError): - base64.b64encode(b, wrapcol='8') - with self.assertRaises(TypeError): - base64.b64encode(b, wrapcol=None) - eq(base64.b64encode(b'', wrapcol=0), b'') - eq(base64.b64encode(b'', wrapcol=8), b'') + func(data, wrapcol='80') + if func is not base64.b16encode: + with self.assertRaises(TypeError): + func(data, wrapcol=None) + + def test_b64encode_wrapcol(self): + eq = self.assertEqual + b = b'www.python.org' + self._common_test_wrapcol(base64.b64encode, b) + eq(base64.b64encode(b, wrapcol=8), b'd3d3LnB5\ndGhvbi5v\ncmc=') + eq(base64.b64encode(b, wrapcol=11), b'd3d3LnB5\ndGhvbi5v\ncmc=') + eq(base64.b64encode(b, wrapcol=1), b'd3d3\nLnB5\ndGhv\nbi5v\ncmc=') def test_b64decode(self): eq = self.assertEqual @@ -309,6 +333,54 @@ def test_b64decode_padding_error(self): self.assertRaises(binascii.Error, base64.b64decode, b'abc') self.assertRaises(binascii.Error, base64.b64decode, 'abc') + def test_b64decode_padded(self): + b64decode = base64.b64decode + urlsafe_b64decode = base64.urlsafe_b64decode + def check(data, expected, padded=0): + if b'=' in data: + with self.assertRaisesRegex(binascii.Error, 'Padding not allowed'): + b64decode(data, padded=False, validate=True) + self.assertEqual(b64decode(data, padded=False, ignorechars=b'='), + expected) + self.assertEqual(urlsafe_b64decode(data, padded=True), expected) + self.assertEqual(urlsafe_b64decode(data, padded=False), expected) + data = data.replace(b'=', b'') + self.assertEqual(b64decode(data, padded=False), expected) + self.assertEqual(b64decode(data, padded=False, validate=True), + expected) + self.assertEqual(urlsafe_b64decode(data), expected) + + check(b'', b'') + check(b'YQ==', b'a') + check(b'YWI=', b'ab') + check(b'YWJj', b'abc') + check(b'Y=WJj', b'abc') + check(b'YW=Jj', b'abc') + check(b'YWJ=j', b'abc') + + with self.assertRaisesRegex(binascii.Error, 'Incorrect padding'): + urlsafe_b64decode(b'YQ', padded=True) + with self.assertRaisesRegex(binascii.Error, 'Incorrect padding'): + urlsafe_b64decode(b'YWI', padded=True) + + def _common_test_ignorechars(self, func): + eq = self.assertEqual + eq(func(b'', ignorechars=b' \n'), b'') + eq(func(b'', ignorechars=b''), b'') + eq(func(b' \n', ignorechars=b' \n'), b'') + with self.assertRaises(binascii.Error): + func(b' \n', ignorechars=b'') + with self.assertRaises(binascii.Error): + func(b' \n', ignorechars=b' ') + with self.assertRaises(binascii.Error): + func(b' \n', ignorechars=b'\n') + with self.assertRaises(TypeError): + func(b'', ignorechars='') + with self.assertRaises(TypeError): + func(b'', ignorechars=[]) + with self.assertRaises(TypeError): + func(b'', ignorechars=None) + def test_b64decode_invalid_chars(self): # issue 1466065: Test some invalid characters. tests = ((b'%3d==', b'\xdd', b'%$'), @@ -351,12 +423,7 @@ def test_b64decode_invalid_chars(self): r = base64.b64decode(bstr, ignorechars=ignorechars) self.assertEqual(r, res) - with self.assertRaises(TypeError): - base64.b64decode(b'', ignorechars='') - with self.assertRaises(TypeError): - base64.b64decode(b'', ignorechars=[]) - with self.assertRaises(TypeError): - base64.b64decode(b'', ignorechars=None) + self._common_test_ignorechars(base64.b64decode) # Normal alphabet characters will be discarded when alternative given discarded = ("invalid character %a in Base64 data with %s " @@ -469,6 +536,23 @@ def test_b32encode(self): self.check_other_types(base64.b32encode, b'abcd', b'MFRGGZA=') self.check_encode_type_errors(base64.b32encode) + def test_b32encode_padded(self): + b32encode = base64.b32encode + self.assertEqual(b32encode(b'', padded=False), b'') + self.assertEqual(b32encode(b'a', padded=False), b'ME') + self.assertEqual(b32encode(b'ab', padded=False), b'MFRA') + self.assertEqual(b32encode(b'abc', padded=False), b'MFRGG') + self.assertEqual(b32encode(b'abcd', padded=False), b'MFRGGZA') + self.assertEqual(b32encode(b'abcde', padded=False), b'MFRGGZDF') + + def test_b32encode_wrapcol(self): + eq = self.assertEqual + b = b'www.python.org' + self._common_test_wrapcol(base64.b32encode, b) + eq(base64.b32encode(b, wrapcol=16), b'O53XOLTQPF2GQ33O\nFZXXEZY=') + eq(base64.b32encode(b, wrapcol=23), b'O53XOLTQPF2GQ33O\nFZXXEZY=') + eq(base64.b32encode(b, wrapcol=1), b'O53XOLTQ\nPF2GQ33O\nFZXXEZY=') + def test_b32decode(self): eq = self.assertEqual tests = {b'': b'', @@ -504,6 +588,7 @@ def test_b32decode_casefold(self): for data, res in tests.items(): eq(base64.b32decode(data, True), res) + eq(base64.b32decode(data, casefold=True), res) eq(base64.b32decode(data.decode('ascii'), True), res) self.assertRaises(binascii.Error, base64.b32decode, b'me======') @@ -537,6 +622,56 @@ def test_b32decode_map01(self): eq(base64.b32decode(b'M%c023456' % map01, map01=map01), res) eq(base64.b32decode(b'M%cO23456' % map01, map01=map01), res) + def test_b32decode_padded(self): + b32decode = base64.b32decode + def check(data, expected): + if b'=' in data: + with self.assertRaisesRegex(binascii.Error, 'Padding not allowed'): + b32decode(data, padded=False) + self.assertEqual(b32decode(data, padded=False, ignorechars=b'='), + expected) + data = data.replace(b'=', b'') + self.assertEqual(b32decode(data, padded=False), expected) + + check(b'', b'') + check(b'ME======', b'a') + check(b'MFRA====', b'ab') + check(b'MFRGG===', b'abc') + check(b'MFRGGZA=', b'abcd') + check(b'MFRGGZDF', b'abcde') + check(b'M=FRGGZDF', b'abcde') + check(b'MF=RGGZDF', b'abcde') + check(b'MFR=GGZDF', b'abcde') + check(b'MFRG=GZDF', b'abcde') + check(b'MFRGG=ZDF', b'abcde') + check(b'MFRGGZ=DF', b'abcde') + check(b'MFRGGZD=F', b'abcde') + + def test_b32decode_ignorechars(self): + self._common_test_ignorechars(base64.b32decode) + eq = self.assertEqual + eq(base64.b32decode(b'MFRG\n GZDF\n', ignorechars=b' \n'), b'abcde') + eq(base64.b32decode(b'MFRG\n GZDF\n', ignorechars=bytearray(b' \n')), b'abcde') + eq(base64.b32decode(b'M=======FRGGZDF', ignorechars=b'='), b'abcde') + eq(base64.b32decode(b'MF======RGGZDF', ignorechars=b'='), b'abcde') + eq(base64.b32decode(b'MFR=====GGZDF', ignorechars=b'='), b'abcde') + eq(base64.b32decode(b'MFRG====GZDF', ignorechars=b'='), b'abcde') + eq(base64.b32decode(b'MFRGG===ZDF', ignorechars=b'='), b'abcde') + eq(base64.b32decode(b'MFRGGZ==DF', ignorechars=b'='), b'abcde') + eq(base64.b32decode(b'MFRGGZD=F', ignorechars=b'='), b'abcde') + eq(base64.b32decode(b'MFRGGZDF=', ignorechars=b'='), b'abcde') + eq(base64.b32decode(b'MFRA======', ignorechars=b'='), b'ab') + + eq(base64.b32decode(b'mfRggzDfmzTQ====', ignorechars=b'mfgz'), + b'\x88\xe7') + eq(base64.b32decode(b'mfRggzDfmzTQ====', casefold=True, ignorechars=b'mfgz'), + b'abcdefg') + eq(base64.b32decode(b'M0F1R0G1G0Z1D0F1', ignorechars=b'01'), b'abcde') + eq(base64.b32decode(b'M0F1R0G1G0Z1D0F1', map01=b'L', ignorechars=b'01'), + b'c\x8a\xb8\xb8\xcb3\xb2\xb1\xb8\xab') + eq(base64.b32decode(b'M0F1R0G1G0Z1D0F1', map01=b'I', ignorechars=b'01'), + b'c\x8a\x88\xb8\xc83\xb2\x81\xb8\xa8') + def test_b32decode_error(self): tests = [b'abc', b'ABCDEF==', b'==ABCDEF'] prefixes = [b'M', b'ME', b'MFRA', b'MFRGG', b'MFRGGZA', b'MFRGGZDF'] @@ -580,11 +715,21 @@ def test_b32hexencode(self): for to_encode, expected in test_cases: with self.subTest(to_decode=to_encode): self.assertEqual(base64.b32hexencode(to_encode), expected) + self.assertEqual(base64.b32hexencode(to_encode, padded=False), + expected.rstrip(b'=')) def test_b32hexencode_other_types(self): self.check_other_types(base64.b32hexencode, b'abcd', b'C5H66P0=') self.check_encode_type_errors(base64.b32hexencode) + def test_b32hexencode_wrapcol(self): + eq = self.assertEqual + b = b'www.python.org' + self._common_test_wrapcol(base64.b32hexencode, b) + eq(base64.b32hexencode(b, wrapcol=16), b'ETRNEBJGF5Q6GRRE\n5PNN4PO=') + eq(base64.b32hexencode(b, wrapcol=23), b'ETRNEBJGF5Q6GRRE\n5PNN4PO=') + eq(base64.b32hexencode(b, wrapcol=1), b'ETRNEBJG\nF5Q6GRRE\n5PNN4PO=') + def test_b32hexdecode(self): test_cases = [ # to_decode, expected, casefold @@ -619,6 +764,53 @@ def test_b32hexdecode_other_types(self): self.check_other_types(base64.b32hexdecode, b'C5H66===', b'abc') self.check_decode_type_errors(base64.b32hexdecode) + def test_b32hexdecode_padded(self): + b32hexdecode = base64.b32hexdecode + def check(data, expected): + if b'=' in data: + with self.assertRaisesRegex(binascii.Error, 'Padding not allowed'): + b32hexdecode(data, padded=False) + self.assertEqual(b32hexdecode(data, padded=False, ignorechars=b'='), + expected) + data = data.replace(b'=', b'') + self.assertEqual(b32hexdecode(data, padded=False), expected) + + check(b'', b'') + check(b'C4======', b'a') + check(b'C5H0====', b'ab') + check(b'C5H66===', b'abc') + check(b'C5H66P0=', b'abcd') + check(b'C5H66P35', b'abcde') + check(b'C=5H66P35', b'abcde') + check(b'C5=H66P35', b'abcde') + check(b'C5H=66P35', b'abcde') + check(b'C5H6=6P35', b'abcde') + check(b'C5H66=P35', b'abcde') + check(b'C5H66P=35', b'abcde') + check(b'C5H66P3=5', b'abcde') + + def test_b32hexdecode_ignorechars(self): + self._common_test_ignorechars(base64.b32hexdecode) + eq = self.assertEqual + eq(base64.b32hexdecode(b'C5H6\n 6P35\n', ignorechars=b' \n'), b'abcde') + eq(base64.b32hexdecode(b'C5H6\n 6P35\n', ignorechars=bytearray(b' \n')), b'abcde') + eq(base64.b32hexdecode(b'========C5H66P35', ignorechars=b'='), b'abcde') + eq(base64.b32hexdecode(b'C=======5H66P35', ignorechars=b'='), b'abcde') + eq(base64.b32hexdecode(b'C5======H66P35', ignorechars=b'='), b'abcde') + eq(base64.b32hexdecode(b'C5H=====66P35', ignorechars=b'='), b'abcde') + eq(base64.b32hexdecode(b'C5H6====6P35', ignorechars=b'='), b'abcde') + eq(base64.b32hexdecode(b'C5H66===P35', ignorechars=b'='), b'abcde') + eq(base64.b32hexdecode(b'C5H66P==35', ignorechars=b'='), b'abcde') + eq(base64.b32hexdecode(b'C5H66P3=5', ignorechars=b'='), b'abcde') + eq(base64.b32hexdecode(b'C5H66P35=', ignorechars=b'='), b'abcde') + eq(base64.b32hexdecode(b'C5H0======', ignorechars=b'='), b'ab') + + eq(base64.b32hexdecode(b'c5h66p35cpjmg===', ignorechars=b'cghjmp'), + b')\x8c2') + eq(base64.b32hexdecode(b'c5h66p35cpjmg===', casefold=True, + ignorechars=b'cghjmp'), + b'abcdefgh') + def test_b32hexdecode_error(self): tests = [b'abc', b'ABCDEF==', b'==ABCDEF', b'c4======'] prefixes = [b'M', b'ME', b'MFRA', b'MFRGG', b'MFRGGZA', b'MFRGGZDF'] @@ -654,6 +846,14 @@ def test_b16encode(self): b'0102ABCDEF') self.check_encode_type_errors(base64.b16encode) + def test_b16encode_wrapcol(self): + eq = self.assertEqual + b = b'\x01\x02\xab\xcd\xef' + self._common_test_wrapcol(base64.b16encode, b) + eq(base64.b16encode(b, wrapcol=4), b'0102\nABCD\nEF') + eq(base64.b16encode(b, wrapcol=5), b'0102\nABCD\nEF') + eq(base64.b16encode(b, wrapcol=1), b'01\n02\nAB\nCD\nEF') + def test_b16decode(self): eq = self.assertEqual eq(base64.b16decode(b'0102ABCDEF'), b'\x01\x02\xab\xcd\xef') @@ -681,6 +881,14 @@ def test_b16decode(self): # Incorrect "padding" self.assertRaises(binascii.Error, base64.b16decode, '010') + def test_b16decode_ignorechars(self): + self._common_test_ignorechars(base64.b16decode) + eq = self.assertEqual + eq(base64.b16decode(b'A B\nC D\n', ignorechars=b' \n'), b'\xab\xcd') + eq(base64.b16decode(b'A B\nC D\n', ignorechars=bytearray(b' \n')), b'\xab\xcd') + eq(base64.b16decode(b'aBcD', ignorechars=b'ac'), b'\xbd') + eq(base64.b16decode(b'aBcD', casefold=True, ignorechars=b'ac'), b'\xab\xcd') + @hypothesis.given( payload=hypothesis.strategies.binary(), casefold=hypothesis.strategies.booleans()) @@ -737,7 +945,7 @@ def test_a85encode(self): def test_a85encode_wrapcol(self): eq = self.assertEqual b = b'www.python.org' - eq(base64.a85encode(b, wrapcol=0), b'GB\\6`E-ZP=Df.1GEb>') + self._common_test_wrapcol(base64.a85encode, b) eq(base64.a85encode(b, wrapcol=7), b'GB\\6`E-\nZP=Df.1\nGEb>') eq(base64.a85encode(b"\0\0\0\0www.python.org", wrapcol=7), b'zGB\\6`E\n-ZP=Df.\n1GEb>') @@ -750,24 +958,8 @@ def test_a85encode_wrapcol(self): b'G\nB\n\\\n6\n`\nE\n-\nZ\nP\n=\nD\nf\n.\n1\nG\nE\nb\n>') eq(base64.a85encode(b, wrapcol=1, adobe=True), b'<~\nGB\n\\6\n`E\n-Z\nP=\nDf\n.1\nGE\nb>\n~>') - eq(base64.a85encode(b, wrapcol=sys.maxsize), b'GB\\6`E-ZP=Df.1GEb>') - if check_impl_detail(): - eq(base64.a85encode(b, wrapcol=sys.maxsize*2), - b'GB\\6`E-ZP=Df.1GEb>') - with self.assertRaises(OverflowError): - base64.a85encode(b, wrapcol=2**1000) - with self.assertRaises(ValueError): - base64.a85encode(b, wrapcol=-7) with self.assertRaises(ValueError): base64.a85encode(b, wrapcol=-7, adobe=True) - with self.assertRaises(TypeError): - base64.a85encode(b, wrapcol=7.0) - with self.assertRaises(TypeError): - base64.a85encode(b, wrapcol='7') - with self.assertRaises(TypeError): - base64.a85encode(b, wrapcol=None) - eq(base64.a85encode(b'', wrapcol=0), b'') - eq(base64.a85encode(b'', wrapcol=7), b'') eq(base64.a85encode(b'', wrapcol=1, adobe=True), b'<~\n~>') eq(base64.a85encode(b'', wrapcol=3, adobe=True), b'<~\n~>') eq(base64.a85encode(b'', wrapcol=4, adobe=True), b'<~~>') @@ -806,6 +998,14 @@ def test_b85encode(self): self.check_other_types(base64.b85encode, b"www.python.org", b'cXxL#aCvlSZ*DGca%T') + def test_b85encode_wrapcol(self): + eq = self.assertEqual + b = b'www.python.org' + self._common_test_wrapcol(base64.b85encode, b) + eq(base64.b85encode(b, wrapcol=10), b'cXxL#aCvlS\nZ*DGca%T') + eq(base64.b85encode(b, wrapcol=14), b'cXxL#aCvlS\nZ*DGca%T') + eq(base64.b85encode(b, wrapcol=1), b'cXxL#\naCvlS\nZ*DGc\na%T') + def test_z85encode(self): eq = self.assertEqual @@ -841,6 +1041,14 @@ def test_z85encode(self): self.check_other_types(base64.z85encode, b"www.python.org", b'CxXl-AcVLsz/dgCA+t') + def test_z85encode_wrapcol(self): + eq = self.assertEqual + b = b'www.python.org' + self._common_test_wrapcol(base64.z85encode, b) + eq(base64.z85encode(b, wrapcol=10), b'CxXl-AcVLs\nz/dgCA+t') + eq(base64.z85encode(b, wrapcol=14), b'CxXl-AcVLs\nz/dgCA+t') + eq(base64.z85encode(b, wrapcol=1), b'CxXl-\nAcVLs\nz/dgC\nA+t') + def test_a85decode(self): eq = self.assertEqual @@ -1047,24 +1255,20 @@ def test_a85decode_errors(self): self.assertEqual(base64.a85decode(b"a b\nc", ignorechars=b" \n"), b'\xc9\x89') - with self.assertRaises(ValueError): - base64.a85decode(b"a b\nc", ignorechars=b"") - with self.assertRaises(ValueError): - base64.a85decode(b"a b\nc", ignorechars=b" ") - with self.assertRaises(ValueError): - base64.a85decode(b"a b\nc", ignorechars=b"\n") - with self.assertRaises(TypeError): - base64.a85decode(b"a b\nc", ignorechars=" \n") - with self.assertRaises(TypeError): - base64.a85decode(b"a b\nc", ignorechars=None) + self._common_test_ignorechars(base64.a85decode) def test_b85decode_errors(self): illegal = list(range(33)) + \ list(b'"\',./:[\\]') + \ list(range(128, 256)) for c in illegal: - with self.assertRaises(ValueError, msg=bytes([c])): - base64.b85decode(b'0000' + bytes([c])) + b = bytes([c]) + with self.assertRaises(ValueError, msg=b): + base64.b85decode(b'0000' + b) + self.assertEqual(base64.b85decode(b'0000' + b, ignorechars=b), + b'\x00\x00\x00') + + self._common_test_ignorechars(base64.b85decode) self.assertRaises(ValueError, base64.b85decode, b'|') self.assertRaises(ValueError, base64.b85decode, b'|N') @@ -1077,8 +1281,13 @@ def test_z85decode_errors(self): list(b'"\',;_`|\\~') + \ list(range(128, 256)) for c in illegal: - with self.assertRaises(ValueError, msg=bytes([c])): - base64.z85decode(b'0000' + bytes([c])) + b = bytes([c]) + with self.assertRaises(ValueError, msg=b): + base64.z85decode(b'0000' + b) + self.assertEqual(base64.z85decode(b'0000' + b, ignorechars=b), + b'\x00\x00\x00') + + self._common_test_ignorechars(base64.z85decode) # b'\xff\xff\xff\xff' encodes to b'%nSc0', the following will overflow: self.assertRaises(ValueError, base64.z85decode, b'%') diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 667ec9b5241aa9..81cdacb96241e2 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -10,10 +10,10 @@ # Note: "*_hex" functions are aliases for "(un)hexlify" -b2a_functions = ['b2a_ascii85', 'b2a_base64', 'b2a_base85', +b2a_functions = ['b2a_ascii85', 'b2a_base32', 'b2a_base64', 'b2a_base85', 'b2a_hex', 'b2a_qp', 'b2a_uu', 'hexlify'] -a2b_functions = ['a2b_ascii85', 'a2b_base64', 'a2b_base85', +a2b_functions = ['a2b_ascii85', 'a2b_base32', 'a2b_base64', 'a2b_base85', 'a2b_hex', 'a2b_qp', 'a2b_uu', 'unhexlify'] all_functions = a2b_functions + b2a_functions + ['crc32', 'crc_hqx'] @@ -75,6 +75,11 @@ def test_constants(self): b'ABCDEFGHIJKLMNOPQRSTUVWXYZ' b'.-:+=^!/*?&<>()[]{}@%$#') + self.assertEqual(binascii.BASE32_ALPHABET, + b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567') + self.assertEqual(binascii.BASE32HEX_ALPHABET, + b'0123456789ABCDEFGHIJKLMNOPQRSTUV') + def test_functions(self): # Check presence of all functions for name in all_functions: @@ -154,20 +159,20 @@ def addnoise(line): def test_base64_bad_padding(self): # Test malformed padding def _assertRegexTemplate(assert_regex, data, - non_strict_mode_expected_result): + non_strict_mode_expected_result, **kwargs): data = self.type2test(data) with self.assertRaisesRegex(binascii.Error, assert_regex): - binascii.a2b_base64(data, strict_mode=True) + binascii.a2b_base64(data, strict_mode=True, **kwargs) self.assertEqual(binascii.a2b_base64(data, strict_mode=False), non_strict_mode_expected_result) self.assertEqual(binascii.a2b_base64(data, strict_mode=True, - ignorechars=b'='), + ignorechars=b' ='), non_strict_mode_expected_result) self.assertEqual(binascii.a2b_base64(data), non_strict_mode_expected_result) - def assertLeadingPadding(*args): - _assertRegexTemplate(r'(?i)Leading padding', *args) + def assertLeadingPadding(*args, **kwargs): + _assertRegexTemplate(r'(?i)Leading padding', *args, **kwargs) def assertDiscontinuousPadding(*args): _assertRegexTemplate(r'(?i)Discontinuous padding', *args) @@ -175,8 +180,11 @@ def assertDiscontinuousPadding(*args): def assertExcessPadding(*args): _assertRegexTemplate(r'(?i)Excess padding', *args) - def assertInvalidLength(*args): - _assertRegexTemplate(r'(?i)Invalid.+number of data characters', *args) + def assertInvalidLength(data, *args, length=None, **kwargs): + if length is None: + length = len(data.split(b'=', 1)[0].replace(b' ', b'')) + assert_regex = fr"(?i)Invalid.+number of data characters \({length}\)" + _assertRegexTemplate(assert_regex, data, *args, **kwargs) assertExcessPadding(b'ab===', b'i') assertExcessPadding(b'ab====', b'i') @@ -195,12 +203,16 @@ def assertInvalidLength(*args): assertLeadingPadding(b'===abcd', b'i\xb7\x1d') assertLeadingPadding(b'====abcd', b'i\xb7\x1d') assertLeadingPadding(b'=====abcd', b'i\xb7\x1d') + assertLeadingPadding(b' =abcd', b'i\xb7\x1d', ignorechars=b' ') assertInvalidLength(b'a=b==', b'i') assertInvalidLength(b'a=bc=', b'i\xb7') assertInvalidLength(b'a=bc==', b'i\xb7') assertInvalidLength(b'a=bcd', b'i\xb7\x1d') assertInvalidLength(b'a=bcd=', b'i\xb7\x1d') + assertInvalidLength(b' a=b==', b'i', ignorechars=b' ') + assertInvalidLength(b'abcde=f==', b'i\xb7\x1dy') + assertInvalidLength(b' abcde=f==', b'i\xb7\x1dy', ignorechars=b' ') assertDiscontinuousPadding(b'ab=c=', b'i\xb7') assertDiscontinuousPadding(b'ab=cd', b'i\xb7\x1d') @@ -221,8 +233,52 @@ def assertInvalidLength(*args): assertExcessPadding(b'abcd====efgh', b'i\xb7\x1dy\xf8!') assertExcessPadding(b'abcd=====efgh', b'i\xb7\x1dy\xf8!') + def test_a2b_base64_padded(self): + a2b_base64 = binascii.a2b_base64 + t = self.type2test + def check(data, expected): + if b'=' in data: + with self.assertRaisesRegex(binascii.Error, 'Padding not allowed'): + a2b_base64(t(data), padded=False, strict_mode=True) + self.assertEqual(a2b_base64(t(data), padded=False, ignorechars=b'='), + expected) + data = data.replace(b'=', b'') + self.assertEqual(a2b_base64(t(data), padded=False), expected) + self.assertEqual(a2b_base64(t(data), padded=False, strict_mode=True), + expected) + + check(b'', b'') + check(b'YQ==', b'a') + check(b'YWI=', b'ab') + check(b'YWJj', b'abc') + check(b'Y=WJj', b'abc') + check(b'YW=Jj', b'abc') + check(b'YWJ=j', b'abc') + + def _common_test_ignorechars(self, func): + eq = self.assertEqual + empty = self.type2test(b'') + data = self.type2test(b'\n \n') + ignorechars = self.type2test(b' \n') + eq(func(empty, ignorechars=ignorechars), b'') + eq(func(empty, ignorechars=empty), b'') + eq(func(data, ignorechars=ignorechars), b'') + with self.assertRaises(binascii.Error): + func(data, ignorechars=empty) + with self.assertRaises(binascii.Error): + func(data, ignorechars=ignorechars[1:]) + with self.assertRaises(binascii.Error): + func(data, ignorechars=ignorechars[:-1]) + with self.assertRaises(TypeError): + func(empty, ignorechars='') + with self.assertRaises(TypeError): + func(empty, ignorechars=[]) + with self.assertRaises(TypeError): + func(empty, ignorechars=None) + def test_base64_invalidchars(self): # Test non-base64 data exceptions + self._common_test_ignorechars(binascii.a2b_base64) def assertNonBase64Data(data, expected, ignorechars): data = self.type2test(data) assert_regex = r'(?i)Only base64 data' @@ -248,6 +304,12 @@ def assertNonBase64Data(data, expected, ignorechars): assertNonBase64Data(b'a\nb==', b'i', ignorechars=bytearray(b'\n')) assertNonBase64Data(b'a\nb==', b'i', ignorechars=memoryview(b'\n')) + self.assertEqual(binascii.a2b_base64(b'+A-/B_', ignorechars=b'+/-_'), + b'\xf8\x0f\xc1') + self.assertEqual(binascii.a2b_base64(b'+A-/B_', ignorechars=b'+/-_', + alphabet=binascii.URLSAFE_BASE64_ALPHABET), + b'\x03\xe0\x7f') + # Same cell in the cache: '\r' >> 3 == '\n' >> 3. data = self.type2test(b'\r\n') with self.assertRaises(binascii.Error): @@ -259,33 +321,23 @@ def assertNonBase64Data(data, expected, ignorechars): binascii.a2b_base64(data, ignorechars=b'*') self.assertEqual(binascii.a2b_base64(data, ignorechars=b'*\n'), b'') - data = self.type2test(b'a\nb==') - with self.assertRaises(TypeError): - binascii.a2b_base64(data, ignorechars='') - with self.assertRaises(TypeError): - binascii.a2b_base64(data, ignorechars=[]) - with self.assertRaises(TypeError): - binascii.a2b_base64(data, ignorechars=None) - def test_base64_excess_data(self): # Test excess data exceptions - def assertExcessData(data, non_strict_expected, - ignore_padchar_expected=None): + def assertExcessData(data, expected): assert_regex = r'(?i)Excess data' data = self.type2test(data) with self.assertRaisesRegex(binascii.Error, assert_regex): binascii.a2b_base64(data, strict_mode=True) self.assertEqual(binascii.a2b_base64(data, strict_mode=False), - non_strict_expected) - if ignore_padchar_expected is not None: - self.assertEqual(binascii.a2b_base64(data, strict_mode=True, - ignorechars=b'='), - ignore_padchar_expected) - self.assertEqual(binascii.a2b_base64(data), non_strict_expected) + expected) + self.assertEqual(binascii.a2b_base64(data, strict_mode=True, + ignorechars=b'='), + expected) + self.assertEqual(binascii.a2b_base64(data), expected) - assertExcessData(b'ab==c', b'i') - assertExcessData(b'ab==cd', b'i', b'i\xb7\x1d') - assertExcessData(b'abc=d', b'i\xb7', b'i\xb7\x1d') + assertExcessData(b'ab==c=', b'i\xb7') + assertExcessData(b'ab==cd', b'i\xb7\x1d') + assertExcessData(b'abc=d', b'i\xb7\x1d') def test_base64errors(self): # Test base64 with invalid padding @@ -492,8 +544,32 @@ def assertInvalidChar(data, **kwargs): assertInvalidChar(b"\tFCb", ignorechars=b"\n") assertInvalidChar(b"xxxB\nP\thU'D v/F+", ignorechars=b" \n\tv") + def _common_test_wrapcol(self, func): + eq = self.assertEqual + data = self.data + expected = func(data) + eq(func(data, wrapcol=0), expected) + eq(func(data, wrapcol=8000), expected) + eq(func(b'', wrapcol=0), func(b'')) + eq(func(b'', wrapcol=1), func(b'')) + eq(func(data, wrapcol=sys.maxsize), expected) + if check_impl_detail(): + eq(func(data, wrapcol=sys.maxsize*2), expected) + with self.assertRaises(OverflowError): + func(data, wrapcol=2**1000) + with self.assertRaises(ValueError): + func(data, wrapcol=-80) + with self.assertRaises(TypeError): + func(data, wrapcol=80.0) + with self.assertRaises(TypeError): + func(data, wrapcol='80') + with self.assertRaises(TypeError): + func(data, wrapcol=None) + def test_ascii85_wrapcol(self): # Test Ascii85 splitting lines + self._common_test_wrapcol(binascii.b2a_ascii85) + def assertEncode(a_expected, data, n, adobe=False): b = self.type2test(data) a = binascii.b2a_ascii85(b, adobe=adobe, wrapcol=n) @@ -632,6 +708,16 @@ def assertOverflow(data): assertOverflow(b"|NsC0~") assertOverflow(b"|NsC0|NsC0|NsD0") + def test_base85_wrapcol(self): + self._common_test_wrapcol(binascii.b2a_base85) + b = self.type2test(b'www.python.org') + self.assertEqual(binascii.b2a_base85(b, wrapcol=10), + b'cXxL#aCvlS\nZ*DGca%T') + self.assertEqual(binascii.b2a_base85(b, wrapcol=14), + b'cXxL#aCvlS\nZ*DGca%T') + self.assertEqual(binascii.b2a_base85(b, wrapcol=1), + b'cXxL#\naCvlS\nZ*DGc\na%T') + def test_base85_pad(self): # Test base85 with encode padding rawdata = b"n1n3Tee\n ch@rAc\te\r$" @@ -643,6 +729,17 @@ def test_base85_pad(self): b_pad_expected = b + b"\0" * padding self.assertEqual(b_pad, b_pad_expected) + def test_base85_ignorechars(self): + a2b_base85 = binascii.a2b_base85 + self._common_test_ignorechars(a2b_base85) + eq = self.assertEqual + eq(a2b_base85(b'VPa\n !s\n', ignorechars=b' \n'), b'abcd') + eq(a2b_base85(b'VPa\n !s\n', ignorechars=bytearray(b' \n')), b'abcd') + + eq(a2b_base85(b'A~[B];C', ignorechars=b';[]~'), b'"1\xa3\x15') + eq(a2b_base85(b'A~[B];C', ignorechars=b';[]~', + alphabet=binascii.Z85_ALPHABET), b'r\xd8dv') + def test_base85_alphabet(self): alphabet = (b'0123456789abcdefghijklmnopqrstuvwxyz' b'ABCDEFGHIJKLMNOPQRSTUVWXYZ.-:+=^!/*?&<>()[]{}@%$#') @@ -670,6 +767,266 @@ def test_base85_alphabet(self): with self.assertRaises(TypeError): binascii.a2b_base64(data, alphabet=bytearray(alphabet)) + def test_base32_valid(self): + # Test base32 with valid data + lines = [] + step = 0 + i = 0 + while i < len(self.rawdata): + b = self.type2test(self.rawdata[i:i + step]) + a = binascii.b2a_base32(b) + lines.append(a) + i += step + step += 1 + res = bytes() + for line in lines: + a = self.type2test(line) + b = binascii.a2b_base32(a) + res += b + self.assertEqual(res, self.rawdata) + + def test_base32_errors(self): + def _fixPadding(data): + fixed = data.replace(b"=", b"") + len_8 = len(fixed) % 8 + p = 8 - len_8 if len_8 else 0 + return fixed + b"=" * p + + def _assertRegexTemplate(assert_regex, data, good_padding_result=None, **kwargs): + with self.assertRaisesRegex(binascii.Error, assert_regex): + binascii.a2b_base32(self.type2test(data), **kwargs) + if good_padding_result: + fixed = self.type2test(_fixPadding(data)) + self.assertEqual(binascii.a2b_base32(fixed), good_padding_result) + + def assertNonBase32Data(*args): + _assertRegexTemplate(r"(?i)Only base32 data", *args) + + def assertExcessData(*args): + _assertRegexTemplate(r"(?i)Excess data", *args) + + def assertExcessPadding(*args): + _assertRegexTemplate(r"(?i)Excess padding", *args) + + def assertLeadingPadding(*args, **kwargs): + _assertRegexTemplate(r"(?i)Leading padding", *args, **kwargs) + + def assertIncorrectPadding(*args): + _assertRegexTemplate(r"(?i)Incorrect padding", *args) + + def assertDiscontinuousPadding(*args): + _assertRegexTemplate(r"(?i)Discontinuous padding", *args) + + def assertInvalidLength(data, *args, length=None, **kwargs): + if length is None: + length = len(data.split(b'=', 1)[0].replace(b' ', b'')) + assert_regex = fr"(?i)Invalid.+number of data characters \({length}\)" + _assertRegexTemplate(assert_regex, data, *args, **kwargs) + + assertNonBase32Data(b"a") + assertNonBase32Data(b"AA-") + assertNonBase32Data(b"ABCDE==!") + assertNonBase32Data(b"ab:(){:|:&};:==") + + assertExcessData(b"AB======C") + assertExcessData(b"AB======CD") + assertExcessData(b"ABCD====E") + assertExcessData(b"ABCDE===FGH") + assertExcessData(b"ABCDEFG=H") + assertExcessData(b"432Z====55555555") + + assertExcessData(b"BE======EF", b"\t\x08") + assertExcessData(b"BEEF====C", b"\t\x08Q") + assertExcessData(b"BEEFC===AK", b"\t\x08Q\x01") + assertExcessData(b"BEEFCAK=E", b"\t\x08Q\x01D") + + assertExcessPadding(b"BE=======", b"\t") + assertExcessPadding(b"BE========", b"\t") + assertExcessPadding(b"BEEF=====", b"\t\x08") + assertExcessPadding(b"BEEF======", b"\t\x08") + assertExcessPadding(b"BEEFC====", b"\t\x08Q") + assertExcessPadding(b"BEEFC=====", b"\t\x08Q") + assertExcessPadding(b"BEEFCAK==", b"\t\x08Q\x01") + assertExcessPadding(b"BEEFCAK===", b"\t\x08Q\x01") + assertExcessPadding(b"BEEFCAKE=", b"\t\x08Q\x01D") + assertExcessPadding(b"BEEFCAKE==", b"\t\x08Q\x01D") + assertExcessPadding(b"BEEFCAKE===", b"\t\x08Q\x01D") + assertExcessPadding(b"BEEFCAKE====", b"\t\x08Q\x01D") + assertExcessPadding(b"BEEFCAKE=====", b"\t\x08Q\x01D") + assertExcessPadding(b"BEEFCAKE======", b"\t\x08Q\x01D") + assertExcessPadding(b"BEEFCAKE=======", b"\t\x08Q\x01D") + assertExcessPadding(b"BEEFCAKE========", b"\t\x08Q\x01D") + assertExcessPadding(b"BEEFCAKE=========", b"\t\x08Q\x01D") + + assertLeadingPadding(b"=", b"") + assertLeadingPadding(b"==", b"") + assertLeadingPadding(b"===", b"") + assertLeadingPadding(b"====", b"") + assertLeadingPadding(b"=====", b"") + assertLeadingPadding(b"======", b"") + assertLeadingPadding(b"=======", b"") + assertLeadingPadding(b"========", b"") + assertLeadingPadding(b"=========", b"") + assertLeadingPadding(b"=BEEFCAKE", b"\t\x08Q\x01D") + assertLeadingPadding(b"==BEEFCAKE", b"\t\x08Q\x01D") + assertLeadingPadding(b"===BEEFCAKE", b"\t\x08Q\x01D") + assertLeadingPadding(b"====BEEFCAKE", b"\t\x08Q\x01D") + assertLeadingPadding(b"=====BEEFCAKE", b"\t\x08Q\x01D") + assertLeadingPadding(b"======BEEFCAKE", b"\t\x08Q\x01D") + assertLeadingPadding(b"=======BEEFCAKE", b"\t\x08Q\x01D") + assertLeadingPadding(b"========BEEFCAKE", b"\t\x08Q\x01D") + assertLeadingPadding(b"=========BEEFCAKE", b"\t\x08Q\x01D") + assertLeadingPadding(b" =BEEFCAKE", ignorechars=b' ') + + assertIncorrectPadding(b"AB") + assertIncorrectPadding(b"ABCD") + assertIncorrectPadding(b"ABCDE") + assertIncorrectPadding(b"ABCDEFG") + + assertIncorrectPadding(b"BE=", b"\t") + assertIncorrectPadding(b"BE==", b"\t") + assertIncorrectPadding(b"BE===", b"\t") + assertIncorrectPadding(b"BE====", b"\t") + assertIncorrectPadding(b"BE=====", b"\t") + assertIncorrectPadding(b"BEEF=", b"\t\x08") + assertIncorrectPadding(b"BEEF==", b"\t\x08") + assertIncorrectPadding(b"BEEF===", b"\t\x08") + assertIncorrectPadding(b"BEEFC=", b"\t\x08Q") + assertIncorrectPadding(b"BEEFC==", b"\t\x08Q") + + assertDiscontinuousPadding(b"BE=EF===", b"\t\x08") + assertDiscontinuousPadding(b"BE==EF==", b"\t\x08") + assertDiscontinuousPadding(b"BEEF=C==", b"\t\x08Q") + assertDiscontinuousPadding(b"BEEFC=AK", b"\t\x08Q\x01") + + assertInvalidLength(b"A") + assertInvalidLength(b"ABC") + assertInvalidLength(b"ABCDEF") + assertInvalidLength(b"ABCDEFGHI") + assertInvalidLength(b"ABCDEFGHIJK") + assertInvalidLength(b"ABCDEFGHIJKLMN") + + assertInvalidLength(b"A=") + assertInvalidLength(b"A==") + assertInvalidLength(b"A===") + assertInvalidLength(b"A====") + assertInvalidLength(b"A=====") + assertInvalidLength(b"A======") + assertInvalidLength(b"ABC=") + assertInvalidLength(b"ABC==") + assertInvalidLength(b"ABC===") + assertInvalidLength(b"ABC====") + assertInvalidLength(b"ABCDEF=") + + assertInvalidLength(b"B=E=====", b"\t") + assertInvalidLength(b"B==E====", b"\t") + assertInvalidLength(b"BEE=F===", b"\t\x08") + assertInvalidLength(b"BEE==F==", b"\t\x08") + assertInvalidLength(b"BEEFCA=K", b"\t\x08Q\x01") + assertInvalidLength(b"BEEFCA=====K", b"\t\x08Q\x01") + + assertInvalidLength(b" A", ignorechars=b' ') + assertInvalidLength(b" ABC", ignorechars=b' ') + assertInvalidLength(b" ABCDEF", ignorechars=b' ') + assertInvalidLength(b" ABCDEFGHI", ignorechars=b' ') + assertInvalidLength(b" ABCDEFGHIJK", ignorechars=b' ') + assertInvalidLength(b" ABCDEFGHIJKLMN", ignorechars=b' ') + assertInvalidLength(b" A=======", ignorechars=b' ') + assertInvalidLength(b" ABC=====", ignorechars=b' ') + assertInvalidLength(b" ABCDEF==", ignorechars=b' ') + + def test_a2b_base32_padded(self): + a2b_base32 = binascii.a2b_base32 + t = self.type2test + def check(data, expected): + if b'=' in data: + with self.assertRaisesRegex(binascii.Error, 'Padding not allowed'): + a2b_base32(t(data), padded=False) + self.assertEqual(a2b_base32(t(data), padded=False, ignorechars=b'='), + expected) + data = data.replace(b'=', b'') + self.assertEqual(a2b_base32(t(data), padded=False), expected) + + check(b'', b'') + check(b'ME======', b'a') + check(b'MFRA====', b'ab') + check(b'MFRGG===', b'abc') + check(b'MFRGGZA=', b'abcd') + check(b'MFRGGZDF', b'abcde') + check(b'M=FRGGZDF', b'abcde') + check(b'MF=RGGZDF', b'abcde') + check(b'MFR=GGZDF', b'abcde') + check(b'MFRG=GZDF', b'abcde') + check(b'MFRGG=ZDF', b'abcde') + check(b'MFRGGZ=DF', b'abcde') + check(b'MFRGGZD=F', b'abcde') + + def test_b2a_base32_padded(self): + b2a_base32 = binascii.b2a_base32 + t = self.type2test + self.assertEqual(b2a_base32(t(b''), padded=False), b'') + self.assertEqual(b2a_base32(t(b'a'), padded=False), b'ME') + self.assertEqual(b2a_base32(t(b'ab'), padded=False), b'MFRA') + self.assertEqual(b2a_base32(t(b'abc'), padded=False), b'MFRGG') + self.assertEqual(b2a_base32(t(b'abcd'), padded=False), b'MFRGGZA') + self.assertEqual(b2a_base32(t(b'abcde'), padded=False), b'MFRGGZDF') + + def test_base32_wrapcol(self): + self._common_test_wrapcol(binascii.b2a_base32) + b = self.type2test(b'www.python.org') + self.assertEqual(binascii.b2a_base32(b, wrapcol=16), + b'O53XOLTQPF2GQ33O\nFZXXEZY=') + self.assertEqual(binascii.b2a_base32(b, wrapcol=23), + b'O53XOLTQPF2GQ33O\nFZXXEZY=') + self.assertEqual(binascii.b2a_base32(b, wrapcol=1), + b'O53XOLTQ\nPF2GQ33O\nFZXXEZY=') + + def test_base32_ignorechars(self): + a2b_base32 = binascii.a2b_base32 + self._common_test_ignorechars(a2b_base32) + eq = self.assertEqual + eq(a2b_base32(b'MFRG\n GZDF\n', ignorechars=b' \n'), b'abcde') + eq(a2b_base32(b'MFRG\n GZDF\n', ignorechars=bytearray(b' \n')), b'abcde') + eq(a2b_base32(b'M=======FRGGZDF', ignorechars=b'='), b'abcde') + eq(a2b_base32(b'MF======RGGZDF', ignorechars=b'='), b'abcde') + eq(a2b_base32(b'MFR=====GGZDF', ignorechars=b'='), b'abcde') + eq(a2b_base32(b'MFRG====GZDF', ignorechars=b'='), b'abcde') + eq(a2b_base32(b'MFRGG===ZDF', ignorechars=b'='), b'abcde') + eq(a2b_base32(b'MFRGGZ==DF', ignorechars=b'='), b'abcde') + eq(a2b_base32(b'MFRGGZD=F', ignorechars=b'='), b'abcde') + eq(a2b_base32(b'MFRGGZDF=', ignorechars=b'='), b'abcde') + eq(a2b_base32(b'MFRA======', ignorechars=b'='), b'ab') + + eq(a2b_base32(b'A1B3C5W7Z9', ignorechars=b'19WZ'), b'\x00v.\xdb\xf9') + eq(a2b_base32(b'A1B3C5W7Z9', ignorechars=b'19WZ', + alphabet=binascii.BASE32HEX_ALPHABET), b'PV6\x14\xe9') + + def test_base32_alphabet(self): + alphabet = b'0Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8Ii9JjKk' + data = self.type2test(self.rawdata) + encoded = binascii.b2a_base32(data, alphabet=alphabet) + trans = bytes.maketrans(binascii.BASE32_ALPHABET, alphabet) + expected = binascii.b2a_base32(data).translate(trans) + self.assertEqual(encoded, expected) + self.assertEqual(binascii.a2b_base32(encoded, alphabet=alphabet), self.rawdata) + self.assertEqual(binascii.b2a_base32(data, alphabet=self.type2test(alphabet)), expected) + + data = self.type2test(b'') + self.assertEqual(binascii.b2a_base32(data, alphabet=alphabet), b'') + self.assertEqual(binascii.a2b_base32(data, alphabet=alphabet), b'') + + for func in binascii.b2a_base32, binascii.a2b_base32: + with self.assertRaises(TypeError): + func(data, alphabet=None) + with self.assertRaises(TypeError): + func(data, alphabet=alphabet.decode()) + with self.assertRaises(ValueError): + func(data, alphabet=alphabet[:-1]) + with self.assertRaises(ValueError): + func(data, alphabet=alphabet+b'?') + with self.assertRaises(TypeError): + binascii.a2b_base32(data, alphabet=bytearray(alphabet)) + def test_uu(self): MAX_UU = 45 for backtick in (True, False): @@ -771,6 +1128,15 @@ def test_hex_separator(self): expected1 = s.hex(':').encode('ascii') self.assertEqual(binascii.b2a_hex(self.type2test(s), ':'), expected1) + def test_hex_ignorechars(self): + a2b_hex = binascii.a2b_hex + self._common_test_ignorechars(a2b_hex) + self._common_test_ignorechars(binascii.unhexlify) + eq = self.assertEqual + eq(a2b_hex(b'A B\nC D\n', ignorechars=b' \n'), b'\xab\xcd') + eq(a2b_hex(b'A B\nC D\n', ignorechars=bytearray(b' \n')), b'\xab\xcd') + eq(a2b_hex(b'aBcD', ignorechars=b'ac'), b'\xab\xcd') + def test_qp(self): type2test = self.type2test a2b_qp = binascii.a2b_qp @@ -947,39 +1313,26 @@ def test_b2a_base64_newline(self): self.assertEqual(binascii.b2a_base64(b, newline=True), b'\n') self.assertEqual(binascii.b2a_base64(b, newline=False), b'') + def test_b2a_base64_padded(self): + b2a_base64 = binascii.b2a_base64 + t = self.type2test + self.assertEqual(b2a_base64(t(b''), padded=False), b'\n') + self.assertEqual(b2a_base64(t(b'a'), padded=False), b'YQ\n') + self.assertEqual(b2a_base64(t(b'ab'), padded=False), b'YWI\n') + self.assertEqual(b2a_base64(t(b'abc'), padded=False), b'YWJj\n') + def test_b2a_base64_wrapcol(self): + self._common_test_wrapcol(binascii.b2a_base64) b = self.type2test(b'www.python.org') - self.assertEqual(binascii.b2a_base64(b), - b'd3d3LnB5dGhvbi5vcmc=\n') - self.assertEqual(binascii.b2a_base64(b, wrapcol=0), - b'd3d3LnB5dGhvbi5vcmc=\n') self.assertEqual(binascii.b2a_base64(b, wrapcol=8), b'd3d3LnB5\ndGhvbi5v\ncmc=\n') self.assertEqual(binascii.b2a_base64(b, wrapcol=11), b'd3d3LnB5\ndGhvbi5v\ncmc=\n') - self.assertEqual(binascii.b2a_base64(b, wrapcol=76), - b'd3d3LnB5dGhvbi5vcmc=\n') self.assertEqual(binascii.b2a_base64(b, wrapcol=8, newline=False), b'd3d3LnB5\ndGhvbi5v\ncmc=') self.assertEqual(binascii.b2a_base64(b, wrapcol=1), b'd3d3\nLnB5\ndGhv\nbi5v\ncmc=\n') - self.assertEqual(binascii.b2a_base64(b, wrapcol=sys.maxsize), - b'd3d3LnB5dGhvbi5vcmc=\n') - if check_impl_detail(): - self.assertEqual(binascii.b2a_base64(b, wrapcol=sys.maxsize*2), - b'd3d3LnB5dGhvbi5vcmc=\n') - with self.assertRaises(OverflowError): - binascii.b2a_base64(b, wrapcol=2**1000) - with self.assertRaises(ValueError): - binascii.b2a_base64(b, wrapcol=-8) - with self.assertRaises(TypeError): - binascii.b2a_base64(b, wrapcol=8.0) - with self.assertRaises(TypeError): - binascii.b2a_base64(b, wrapcol='8') b = self.type2test(b'') - self.assertEqual(binascii.b2a_base64(b), b'\n') - self.assertEqual(binascii.b2a_base64(b, wrapcol=0), b'\n') - self.assertEqual(binascii.b2a_base64(b, wrapcol=8), b'\n') self.assertEqual(binascii.b2a_base64(b, wrapcol=8, newline=False), b'') @hypothesis.given( diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index ab65a44bda6e7e..5a4f031e298c2f 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -66,7 +66,8 @@ '?':0, 'c':0, 'b':0, 'B':0, 'h':0, 'H':0, 'i':0, 'I':0, 'l':0, 'L':0, 'n':0, 'N':0, - 'e':0, 'f':0, 'd':0, 'P':0 + 'e':0, 'f':0, 'd':0, 'P':0, + 'F':0, 'D':0 } # NumPy does not have 'n' or 'N': @@ -92,7 +93,9 @@ 'l':(-(1<<31), 1<<31), 'L':(0, 1<<32), 'q':(-(1<<63), 1<<63), 'Q':(0, 1<<64), 'e':(-65519, 65520), 'f':(-(1<<63), 1<<63), - 'd':(-(1<<1023), 1<<1023) + 'd':(-(1<<1023), 1<<1023), + 'F':(-(1<<63), 1<<63), + 'D':(-(1<<1023), 1<<1023) } def native_type_range(fmt): @@ -107,6 +110,10 @@ def native_type_range(fmt): lh = (-(1<<63), 1<<63) elif fmt == 'd': lh = (-(1<<1023), 1<<1023) + elif fmt == 'F': + lh = (-(1<<63), 1<<63) + elif fmt == 'D': + lh = (-(1<<1023), 1<<1023) else: for exp in (128, 127, 64, 63, 32, 31, 16, 15, 8, 7): try: @@ -175,6 +182,11 @@ def randrange_fmt(mode, char, obj): if char in 'efd': x = struct.pack(char, x) x = struct.unpack(char, x)[0] + if char in 'FD': + y = randrange(*fmtdict[mode][char]) + x = complex(x, y) + x = struct.pack(char, x) + x = struct.unpack(char, x)[0] return x def gen_item(fmt, obj): @@ -3015,7 +3027,7 @@ def test_memoryview_assign(self): m = memoryview(nd) self.assertRaises(TypeError, m.__setitem__, 0, 100) - ex = ndarray(list(range(120)), shape=[1,2,3,4,5], flags=ND_WRITABLE) + ex = ndarray(list(range(144)), shape=[1,2,3,4,6], flags=ND_WRITABLE) m1 = memoryview(ex) for fmt, _range in fmtdict['@'].items(): @@ -3025,7 +3037,7 @@ def test_memoryview_assign(self): continue m2 = m1.cast(fmt) lo, hi = _range - if fmt == 'd' or fmt == 'f': + if fmt in "dfDF": lo, hi = -2**1024, 2**1024 if fmt != 'P': # PyLong_AsVoidPtr() accepts negative numbers self.assertRaises(ValueError, m2.__setitem__, 0, lo-1) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 876ecd4467b0a2..b1cdbe04765ed0 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -551,10 +551,10 @@ def test_hex_separator_basics(self): self.assertEqual(three_bytes.hex('*', -2), 'b901*ef') self.assertEqual(three_bytes.hex(sep=':', bytes_per_sep=2), 'b9:01ef') self.assertEqual(three_bytes.hex(sep='*', bytes_per_sep=-2), 'b901*ef') - for bytes_per_sep in 3, -3, 2**31-1, -(2**31-1): + for bytes_per_sep in 3, -3, sys.maxsize, -sys.maxsize: with self.subTest(bytes_per_sep=bytes_per_sep): self.assertEqual(three_bytes.hex(':', bytes_per_sep), 'b901ef') - for bytes_per_sep in 2**31, -2**31, 2**1000, -2**1000: + for bytes_per_sep in sys.maxsize+1, -sys.maxsize-1, 2**1000, -2**1000: with self.subTest(bytes_per_sep=bytes_per_sep): try: self.assertEqual(three_bytes.hex(':', bytes_per_sep), 'b901ef') @@ -878,6 +878,13 @@ def test_replace(self): self.assertEqual(b.replace(b'i', b'a'), b'massassappa') self.assertEqual(b.replace(b'ss', b'x'), b'mixixippi') + def test_replace_count_keyword(self): + b = self.type2test(b'aa') + self.assertEqual(b.replace(b'a', b'b', count=0), b'aa') + self.assertEqual(b.replace(b'a', b'b', count=1), b'ba') + self.assertEqual(b.replace(b'a', b'b', count=2), b'bb') + self.assertEqual(b.replace(b'a', b'b', count=3), b'bb') + def test_replace_int_error(self): self.assertRaises(TypeError, self.type2test(b'a b').replace, 32, b'') @@ -2363,13 +2370,20 @@ def fixtype(self, obj): contains_bytes = True + def test_mixed_cmp(self): + a = self.type2test(b'ab') + for t in bytes, bytearray, BytesSubclass, ByteArraySubclass: + with self.subTest(t.__name__): + self._assert_cmp(a, t(b'ab'), 0) + self._assert_cmp(a, t(b'a'), 1) + self._assert_cmp(a, t(b'ac'), -1) + class ByteArrayAsStringTest(FixedStringTest, unittest.TestCase): type2test = bytearray class BytesAsStringTest(FixedStringTest, unittest.TestCase): type2test = bytes - class SubclassTest: def test_basic(self): diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index 4967f02b007e06..51ac41e33ac17a 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -13,8 +13,9 @@ from .test_misc import decode_stderr -# Skip this test if the _testcapi module isn't available. +# Skip this test if the _testcapi or _testinternalcapi module isn't available. _testcapi = import_helper.import_module('_testcapi') +_testinternalcapi = import_helper.import_module('_testinternalcapi') NULL = None @@ -108,6 +109,26 @@ def __del__(self): b':7: RuntimeWarning: Testing PyErr_WarnEx', ]) + def test__pyerr_setkeyerror(self): + # Test _PyErr_SetKeyError() + _pyerr_setkeyerror = _testinternalcapi._pyerr_setkeyerror + for arg in ( + "key", + # check that a tuple argument is not unpacked + (1, 2, 3), + # PyErr_SetObject(exc_type, exc_value) uses exc_value if it's + # already an exception, but _PyErr_SetKeyError() always creates a + # new KeyError. + KeyError('arg'), + ): + with self.subTest(arg=arg): + with self.assertRaises(KeyError) as cm: + # Test calling _PyErr_SetKeyError() with an exception set + # to check that the function overrides the current + # exception. + _pyerr_setkeyerror(arg) + self.assertEqual(cm.exception.args, (arg,)) + class Test_FatalError(unittest.TestCase): diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index d3156645eeec2d..d1467caf6880a6 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -803,6 +803,26 @@ def to_digits(num): self.assertEqual(pylongwriter_create(negative, digits), num, (negative, digits)) + @unittest.skipUnless(support.Py_DEBUG, "need a debug build (Py_DEBUG)") + def test_longwriter_finish(self): + # Test PyLongWriter_Create(0, 3, &digits) with PyLongWriter_Finish() + # where the last digit is left uninitialized + pylongwriter_finish_bug = _testcapi.pylongwriter_finish_bug + with self.assertRaises(SystemError) as cm: + pylongwriter_finish_bug() + self.assertEqual(str(cm.exception), + 'PyLongWriter_Finish: digit 2 is uninitialized') + + def test_bug_143050(self): + with support.adjust_int_max_str_digits(0): + # Bug coming from using _pylong.int_from_string(), that + # currently requires > 6000 decimal digits. + int('-' + '0' * 7000, 10) + _testcapi.test_immortal_small_ints() + # Test also nonzero small int + int('-' + '0' * 7000 + '123', 10) + _testcapi.test_immortal_small_ints() + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_module.py b/Lib/test/test_capi/test_module.py index 053e6709cda42e..c32ca1098edc56 100644 --- a/Lib/test/test_capi/test_module.py +++ b/Lib/test/test_capi/test_module.py @@ -25,9 +25,13 @@ def def_and_token(mod): ) class TestModFromSlotsAndSpec(unittest.TestCase): - @requires_gil_enabled("empty slots re-enable GIL") def test_empty(self): - mod = _testcapi.module_from_slots_empty(FakeSpec()) + with self.assertRaises(SystemError): + _testcapi.module_from_slots_empty(FakeSpec()) + + @requires_gil_enabled("minimal slots re-enable GIL") + def test_minimal(self): + mod = _testcapi.module_from_slots_minimal(FakeSpec()) self.assertIsInstance(mod, types.ModuleType) self.assertEqual(def_and_token(mod), (0, 0)) self.assertEqual(mod.__name__, 'testmod') @@ -159,6 +163,16 @@ def test_null_def_slot(self): self.assertIn(name, str(cm.exception)) self.assertIn("NULL", str(cm.exception)) + def test_bad_abiinfo(self): + """Slots that incompatible ABI is rejected""" + with self.assertRaises(ImportError) as cm: + _testcapi.module_from_bad_abiinfo(FakeSpec()) + + def test_multiple_abiinfo(self): + """Slots that Py_mod_abiinfo can be repeated""" + mod = _testcapi.module_from_multiple_abiinfo(FakeSpec()) + self.assertEqual(mod.__name__, 'testmod') + def test_def_multiple_exec(self): """PyModule_Exec runs all exec slots of PyModuleDef-defined module""" mod = _testcapi.module_from_def_multiple_exec(FakeSpec()) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index bd9bc8a1a533c3..56f90194b480a1 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -20,6 +20,18 @@ #For test of issue 136154 GLOBAL_136154 = 42 +# For frozendict JIT tests +FROZEN_DICT_CONST = frozendict(x=1, y=2) + +# For frozenset JIT tests +FROZEN_SET_CONST = frozenset({1, 2, 3}) + +class _GenericKey: + pass + +_GENERIC_KEY = _GenericKey() + + @contextlib.contextmanager def clear_executors(func): # Clear executors in func before and after running a block @@ -2003,8 +2015,8 @@ def f(n): self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertEqual(uops.count("_GUARD_NOS_DICT"), 0) - self.assertEqual(uops.count("_STORE_SUBSCR_DICT"), 1) - self.assertEqual(uops.count("_BINARY_OP_SUBSCR_DICT"), 1) + self.assertEqual(uops.count("_STORE_SUBSCR_DICT_KNOWN_HASH"), 1) + self.assertEqual(uops.count("_BINARY_OP_SUBSCR_DICT_KNOWN_HASH"), 1) def test_remove_guard_for_known_type_list(self): def f(n): @@ -2029,6 +2041,125 @@ def f(n): self.assertEqual(uops.count("_BINARY_OP_SUBSCR_LIST_INT"), 1) self.assertEqual(uops.count("_TO_BOOL_LIST"), 1) + def test_unique_tuple_unpack(self): + def f(n): + def four_tuple(x): + return (x, x, x, x) + hits = 0 + for i in range(n): + w, x, y, z = four_tuple(1) + hits += w + x + y + z + return hits + + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 4) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertIn("_BUILD_TUPLE", uops) + self.assertIn("_UNPACK_SEQUENCE_UNIQUE_TUPLE", uops) + self.assertNotIn("_UNPACK_SEQUENCE_TUPLE", uops) + + def test_non_unique_tuple_unpack(self): + def f(n): + def four_tuple(x): + return (x, x, x, x) + hits = 0 + for i in range(n): + t = four_tuple(1) + w, x, y, z = t + hits += w + x + y + z + return hits + + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 4) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertIn("_BUILD_TUPLE", uops) + self.assertIn("_UNPACK_SEQUENCE_TUPLE", uops) + self.assertNotIn("_UNPACK_SEQUENCE_UNIQUE_TUPLE", uops) + + def test_unique_three_tuple_unpack(self): + def f(n): + def three_tuple(x): + return (x, x, x) + hits = 0 + for i in range(n): + x, y, z = three_tuple(1) + hits += x + y + z + return hits + + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 3) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertIn("_BUILD_TUPLE", uops) + self.assertIn("_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE", uops) + self.assertNotIn("_UNPACK_SEQUENCE_TUPLE", uops) + + def test_non_unique_three_tuple_unpack(self): + def f(n): + def three_tuple(x): + return (x, x, x) + hits = 0 + for i in range(n): + t = three_tuple(1) + x, y, z = t + hits += x + y + z + return hits + + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 3) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertIn("_BUILD_TUPLE", uops) + self.assertIn("_UNPACK_SEQUENCE_TUPLE", uops) + self.assertNotIn("_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE", uops) + + def test_unique_two_tuple_unpack(self): + def f(n): + def two_tuple(x): + return (x, x) + hits = 0 + for i in range(n): + x, y = two_tuple(1) + hits += x + y + return hits + + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 2) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertIn("_BUILD_TUPLE", uops) + self.assertIn("_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE", uops) + self.assertNotIn("_UNPACK_SEQUENCE_TWO_TUPLE", uops) + self.assertNotIn("_UNPACK_SEQUENCE_TUPLE", uops) + + def test_non_unique_two_tuple_unpack(self): + def f(n): + def two_tuple(x): + return (x, x) + hits = 0 + for i in range(n): + tt = two_tuple(1) + x, y = tt + hits += x + y + return hits + + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 2) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertIn("_BUILD_TUPLE", uops) + self.assertIn("_UNPACK_SEQUENCE_TWO_TUPLE", uops) + self.assertNotIn("_UNPACK_SEQUENCE_TUPLE", uops) + self.assertNotIn("_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE", uops) + def test_remove_guard_for_known_type_set(self): def f(n): x = 0 @@ -2041,7 +2172,8 @@ def f(n): self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertNotIn("_GUARD_TOS_ANY_SET", uops) - self.assertIn("_CONTAINS_OP_SET", uops) + # _CONTAINS_OP_SET is constant-folded away for frozenset literals + self.assertIn("_INSERT_2_LOAD_CONST_INLINE_BORROW", uops) def test_remove_guard_for_known_type_tuple(self): def f(n): @@ -2189,9 +2321,46 @@ def testfunc(n): self.assertEqual(res, TIER2_THRESHOLD) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_BINARY_OP_SUBSCR_DICT", uops) + self.assertIn("_BINARY_OP_SUBSCR_DICT_KNOWN_HASH", uops) self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) + def test_binary_op_subscr_dict_known_hash(self): + # str, int, bytes, float, complex, tuple and any python object which has generic hash + def testfunc(n): + x = 0 + d = {'a': 1, 1: 2, b'b': 3, (1, 2): 4, _GENERIC_KEY: 5, 1.5: 6, 1+2j: 7} + for _ in range(n): + x += d['a'] + d[1] + d[b'b'] + d[(1, 2)] + d[_GENERIC_KEY] + d[1.5] + d[1+2j] + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, 28 * TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_SUBSCR_DICT_KNOWN_HASH", uops) + self.assertNotIn("_BINARY_OP_SUBSCR_DICT", uops) + + def test_store_subscr_dict_known_hash(self): + # str, int, bytes, float, complex, tuple and any python object which has generic hash + def testfunc(n): + d = {'a': 0, 1: 0, b'b': 0, (1, 2): 0, _GENERIC_KEY: 0, 1.5: 0, 1+2j: 0} + for _ in range(n): + d['a'] += 1 + d[1] += 2 + d[b'b'] += 3 + d[(1, 2)] += 4 + d[_GENERIC_KEY] += 5 + d[1.5] += 6 + d[1+2j] += 7 + return d['a'] + d[1] + d[b'b'] + d[(1, 2)] + d[_GENERIC_KEY] + d[1.5] + d[1+2j] + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, 28 * TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_STORE_SUBSCR_DICT_KNOWN_HASH", uops) + self.assertNotIn("_STORE_SUBSCR_DICT", uops) + def test_contains_op(self): def testfunc(n): x = 0 @@ -2545,9 +2714,40 @@ def testfunc(n): self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertIn("_CALL_BUILTIN_O", uops) + self.assertNotIn("_GUARD_CALLABLE_BUILTIN_O", uops) self.assertIn("_POP_TOP_NOP", uops) self.assertLessEqual(count_ops(ex, "_POP_TOP"), 4) + def test_call_builtin_fast(self): + def testfunc(n): + x = 0 + for _ in range(n): + y = divmod(10, 3) + x += y[0] + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 3) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CALL_BUILTIN_FAST", uops) + self.assertNotIn("_GUARD_CALLABLE_BUILTIN_FAST", uops) + + def test_call_builtin_fast_with_keywords(self): + def testfunc(n): + x = 0 + for _ in range(n): + y = sorted([3, 1, 2]) + x += y[0] + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CALL_BUILTIN_FAST_WITH_KEYWORDS", uops) + self.assertNotIn("_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS", uops) + def test_call_method_descriptor_o(self): def testfunc(n): x = 0 @@ -2561,10 +2761,79 @@ def testfunc(n): self.assertEqual(res, TIER2_THRESHOLD) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_CALL_METHOD_DESCRIPTOR_O", uops) + self.assertIn("_CALL_METHOD_DESCRIPTOR_O_INLINE", uops) + self.assertNotIn("_CALL_METHOD_DESCRIPTOR_O", uops) + self.assertNotIn("_GUARD_CALLABLE_METHOD_DESCRIPTOR_O", uops) self.assertIn("_POP_TOP_NOP", uops) self.assertLessEqual(count_ops(ex, "_POP_TOP"), 4) + def test_call_method_descriptor_noargs(self): + def testfunc(n): + x = 0 + for _ in range(n): + y = "hello" + z = y.upper() + x += len(z) + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 5) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE", uops) + self.assertNotIn("_CALL_METHOD_DESCRIPTOR_NOARGS", uops) + self.assertNotIn("_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS", uops) + + def test_call_method_descriptor_fast(self): + def testfunc(n): + x = 0 + for _ in range(n): + y = (1, 2, 3) + z = y.index(2) + x += z + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CALL_METHOD_DESCRIPTOR_FAST_INLINE", uops) + self.assertNotIn("_CALL_METHOD_DESCRIPTOR_FAST", uops) + self.assertNotIn("_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST", uops) + + def test_call_method_descriptor_fast_with_keywords(self): + def testfunc(n): + x = 0 + for _ in range(n): + y = "hello world" + a, b = y.split() + x += len(a) + return x + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 5) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE", uops) + self.assertNotIn("_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", uops) + self.assertNotIn("_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", uops) + + def test_check_recursion_limit_deduplicated(self): + def testfunc(n): + x = 0 + for _ in range(n): + y = "hello" + a = y.upper() + b = y.lower() + x += len(a) + x += len(b) + return x + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 10) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE", uops) + self.assertEqual(count_ops(ex, "_CHECK_RECURSION_LIMIT"), 1) + def test_call_intrinsic_1(self): def testfunc(n): x = 0 @@ -2580,6 +2849,22 @@ def testfunc(n): self.assertEqual(count_ops(ex, "_POP_TOP_NOP"), 1) self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) + def test_call_intrinsic_2(self): + def testfunc(n): + x = 0 + for _ in range(n): + def test_testfunc[T](n): + pass + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, 0) + uops = get_opnames(ex) + + self.assertIn("_CALL_INTRINSIC_2", uops) + self.assertGreaterEqual(count_ops(ex, "_POP_TOP_NOP"), 2) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 4) + def test_get_len_with_const_tuple(self): def testfunc(n): x = 0.0 @@ -2673,9 +2958,6 @@ def testfunc(n): self.assertEqual(res, sum(range(TIER2_THRESHOLD))) uops = get_opnames(ex) self.assertIn("_CALL_LIST_APPEND", uops) - # We should remove these in the future - self.assertIn("_GUARD_NOS_LIST", uops) - self.assertIn("_GUARD_CALLABLE_LIST_APPEND", uops) def test_call_list_append_pop_top(self): def testfunc(n): @@ -2898,6 +3180,21 @@ def f(n): self.assertNotIn("_LOAD_ATTR_METHOD_NO_DICT", uops) self.assertNotIn("_LOAD_ATTR_METHOD_LAZY_DICT", uops) + def test_cached_attributes_fixed_version_tag(self): + def f(n): + c = 1 + x = 0 + for _ in range(n): + x += c.bit_length() + return x + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + self.assertEqual(res, TIER2_THRESHOLD) + uops = get_opnames(ex) + self.assertNotIn("_LOAD_ATTR_METHOD_NO_DICT", uops) + self.assertNotIn("_LOAD_CONST_UNDER_INLINE", uops) + self.assertIn("_LOAD_CONST_UNDER_INLINE_BORROW", uops) + def test_store_fast_refcount_elimination(self): def foo(x): # Since x is known to be @@ -2953,6 +3250,319 @@ def testfunc(args): uops = get_opnames(ex) self.assertIn("_POP_TOP_NOP", uops) + def test_float_add_inplace_unique_lhs(self): + # a * b produces a unique float; adding c reuses it in place + def testfunc(args): + a, b, c, n = args + total = 0.0 + for _ in range(n): + total += a * b + c + return total + + res, ex = self._run_with_optimizer(testfunc, (2.0, 3.0, 4.0, TIER2_THRESHOLD)) + self.assertAlmostEqual(res, TIER2_THRESHOLD * 10.0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_ADD_FLOAT_INPLACE", uops) + + def test_float_add_inplace_unique_rhs(self): + # a * b produces a unique float on the right side of + + def testfunc(args): + a, b, c, n = args + total = 0.0 + for _ in range(n): + total += c + a * b + return total + + res, ex = self._run_with_optimizer(testfunc, (2.0, 3.0, 4.0, TIER2_THRESHOLD)) + self.assertAlmostEqual(res, TIER2_THRESHOLD * 10.0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT", uops) + + def test_float_add_no_inplace_non_unique(self): + # Both operands of a + b are locals — neither is unique, + # so the first add is regular. But total += (a+b) has a + # unique RHS, so it uses _INPLACE_RIGHT. + def testfunc(args): + a, b, n = args + total = 0.0 + for _ in range(n): + total += a + b + return total + + res, ex = self._run_with_optimizer(testfunc, (2.0, 3.0, TIER2_THRESHOLD)) + self.assertAlmostEqual(res, TIER2_THRESHOLD * 5.0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + # a + b: both are locals, no inplace + self.assertIn("_BINARY_OP_ADD_FLOAT", uops) + # total += result: result is unique RHS + self.assertIn("_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT", uops) + # No LHS inplace variant for the first add + self.assertNotIn("_BINARY_OP_ADD_FLOAT_INPLACE", uops) + + def test_float_subtract_inplace_unique_lhs(self): + # a * b produces a unique float; subtracting c reuses it + def testfunc(args): + a, b, c, n = args + total = 0.0 + for _ in range(n): + total += a * b - c + return total + + res, ex = self._run_with_optimizer(testfunc, (2.0, 3.0, 1.0, TIER2_THRESHOLD)) + self.assertAlmostEqual(res, TIER2_THRESHOLD * 5.0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_SUBTRACT_FLOAT_INPLACE", uops) + + def test_float_subtract_inplace_unique_rhs(self): + # a * b produces a unique float on the right of -; + # result is c - (a * b), must get the sign correct + def testfunc(args): + a, b, c, n = args + total = 0.0 + for _ in range(n): + total += c - a * b + return total + + res, ex = self._run_with_optimizer(testfunc, (2.0, 3.0, 1.0, TIER2_THRESHOLD)) + self.assertAlmostEqual(res, TIER2_THRESHOLD * -5.0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT", uops) + + def test_float_multiply_inplace_unique_lhs(self): + # (a + b) produces a unique float; multiplying by c reuses it + def testfunc(args): + a, b, c, n = args + total = 0.0 + for _ in range(n): + total += (a + b) * c + return total + + res, ex = self._run_with_optimizer(testfunc, (2.0, 3.0, 4.0, TIER2_THRESHOLD)) + self.assertAlmostEqual(res, TIER2_THRESHOLD * 20.0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_MULTIPLY_FLOAT_INPLACE", uops) + + def test_float_multiply_inplace_unique_rhs(self): + # (a + b) produces a unique float on the right side of * + def testfunc(args): + a, b, c, n = args + total = 0.0 + for _ in range(n): + total += c * (a + b) + return total + + res, ex = self._run_with_optimizer(testfunc, (2.0, 3.0, 4.0, TIER2_THRESHOLD)) + self.assertAlmostEqual(res, TIER2_THRESHOLD * 20.0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT", uops) + + def test_float_inplace_chain_propagation(self): + # a * b + c * d: both products are unique, the + reuses one; + # result of + is also unique for the subsequent += + def testfunc(args): + a, b, c, d, n = args + total = 0.0 + for _ in range(n): + total += a * b + c * d + return total + + res, ex = self._run_with_optimizer(testfunc, (2.0, 3.0, 4.0, 5.0, TIER2_THRESHOLD)) + self.assertAlmostEqual(res, TIER2_THRESHOLD * 26.0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + # The + between the two products should use an inplace variant + inplace_add = ( + "_BINARY_OP_ADD_FLOAT_INPLACE" in uops + or "_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT" in uops + ) + self.assertTrue(inplace_add, + "Expected an inplace add for unique intermediate results") + + def test_float_negate_inplace_unique(self): + # -(a * b): the product is unique, negate it in place + def testfunc(args): + a, b, n = args + total = 0.0 + for _ in range(n): + total += -(a * b) + return total + + res, ex = self._run_with_optimizer(testfunc, (2.0, 3.0, TIER2_THRESHOLD)) + self.assertAlmostEqual(res, TIER2_THRESHOLD * -6.0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_UNARY_NEGATIVE_FLOAT_INPLACE", uops) + + def test_float_negate_no_inplace_non_unique(self): + # -a where a is a local — not unique, no inplace + def testfunc(args): + a, n = args + total = 0.0 + for _ in range(n): + total += -a + return total + + res, ex = self._run_with_optimizer(testfunc, (2.0, TIER2_THRESHOLD)) + self.assertAlmostEqual(res, TIER2_THRESHOLD * -2.0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_UNARY_NEGATIVE_FLOAT_INPLACE", uops) + + def test_int_add_inplace_unique_lhs(self): + # a * b produces a unique compact int; adding c reuses it in place + def testfunc(args): + a, b, c, n = args + total = 0 + for _ in range(n): + total += a * b + c + return total + + res, ex = self._run_with_optimizer(testfunc, (2000, 3, 4000, TIER2_THRESHOLD)) + self.assertEqual(res, TIER2_THRESHOLD * 10000) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_ADD_INT_INPLACE", uops) + + def test_int_add_inplace_unique_rhs(self): + # a * b produces a unique compact int on the right side of + + def testfunc(args): + a, b, c, n = args + total = 0 + for _ in range(n): + total += c + a * b + return total + + res, ex = self._run_with_optimizer(testfunc, (2000, 3, 4000, TIER2_THRESHOLD)) + self.assertEqual(res, TIER2_THRESHOLD * 10000) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_ADD_INT_INPLACE_RIGHT", uops) + + def test_int_add_no_inplace_non_unique(self): + # Both operands of a + b are locals — neither is unique, + # so the first add uses the regular op. But total += (a+b) + # has a unique RHS (result of a+b), so it uses _INPLACE_RIGHT. + def testfunc(args): + a, b, n = args + total = 0 + for _ in range(n): + total += a + b + return total + + res, ex = self._run_with_optimizer(testfunc, (2000, 3000, TIER2_THRESHOLD)) + self.assertEqual(res, TIER2_THRESHOLD * 5000) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + # a + b: both are locals, no inplace + self.assertIn("_BINARY_OP_ADD_INT", uops) + # total += result: result is unique RHS + self.assertIn("_BINARY_OP_ADD_INT_INPLACE_RIGHT", uops) + # No LHS inplace variant for the first add + self.assertNotIn("_BINARY_OP_ADD_INT_INPLACE", uops) + + def test_int_add_inplace_small_int_result(self): + # When the result is a small int, the inplace path falls back + # to _PyCompactLong_Add. Verify correctness (no singleton corruption). + def testfunc(args): + a, b, n = args + total = 0 + for _ in range(n): + total += a * b + 1 # a*b=6, +1=7, small int + return total + + res, ex = self._run_with_optimizer(testfunc, (2, 3, TIER2_THRESHOLD)) + self.assertEqual(res, TIER2_THRESHOLD * 7) + # Verify small int singletons are not corrupted + self.assertEqual(7, 3 + 4) + + def test_int_subtract_inplace_unique_lhs(self): + # a * b produces a unique compact int; subtracting c reuses it + def testfunc(args): + a, b, c, n = args + total = 0 + for _ in range(n): + total += a * b - c + return total + + res, ex = self._run_with_optimizer(testfunc, (2000, 3, 1000, TIER2_THRESHOLD)) + self.assertEqual(res, TIER2_THRESHOLD * 5000) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_SUBTRACT_INT_INPLACE", uops) + + def test_int_subtract_inplace_unique_rhs(self): + # a * b produces a unique compact int on the right of - + def testfunc(args): + a, b, c, n = args + total = 0 + for _ in range(n): + total += c - a * b + return total + + res, ex = self._run_with_optimizer(testfunc, (2000, 3, 10000, TIER2_THRESHOLD)) + self.assertEqual(res, TIER2_THRESHOLD * 4000) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT", uops) + + def test_int_multiply_inplace_unique_lhs(self): + # (a + b) produces a unique compact int; multiplying by c reuses it + def testfunc(args): + a, b, c, n = args + total = 0 + for _ in range(n): + total += (a + b) * c + return total + + res, ex = self._run_with_optimizer(testfunc, (2000, 3000, 4, TIER2_THRESHOLD)) + self.assertEqual(res, TIER2_THRESHOLD * 20000) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_MULTIPLY_INT_INPLACE", uops) + + def test_int_multiply_inplace_unique_rhs(self): + # (a + b) produces a unique compact int on the right side of * + def testfunc(args): + a, b, c, n = args + total = 0 + for _ in range(n): + total += c * (a + b) + return total + + res, ex = self._run_with_optimizer(testfunc, (2000, 3000, 4, TIER2_THRESHOLD)) + self.assertEqual(res, TIER2_THRESHOLD * 20000) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT", uops) + + def test_int_inplace_chain_propagation(self): + # a * b + c * d: both products are unique, the + reuses one; + # result of + is also unique for the subsequent += + def testfunc(args): + a, b, c, d, n = args + total = 0 + for _ in range(n): + total += a * b + c * d + return total + + res, ex = self._run_with_optimizer(testfunc, (2000, 3, 4000, 5, TIER2_THRESHOLD)) + self.assertEqual(res, TIER2_THRESHOLD * 26000) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + inplace_add = ( + "_BINARY_OP_ADD_INT_INPLACE" in uops + or "_BINARY_OP_ADD_INT_INPLACE_RIGHT" in uops + ) + self.assertTrue(inplace_add, + "Expected an inplace add for unique intermediate results") + def test_load_attr_instance_value(self): def testfunc(n): class C(): @@ -3306,7 +3916,7 @@ def testfunc(n): self.assertEqual(res, 10) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_STORE_SUBSCR_DICT", uops) + self.assertIn("_STORE_SUBSCR_DICT_KNOWN_HASH", uops) self.assertLessEqual(count_ops(ex, "_POP_TOP"), 1) self.assertIn("_POP_TOP_NOP", uops) @@ -3764,11 +4374,54 @@ def testfunc(n): self.assertLessEqual(count_ops(ex, "_POP_TOP_INT"), 1) self.assertIn("_POP_TOP_NOP", uops) + def test_binary_subscr_frozendict_lowering(self): + def testfunc(n): + x = 0 + for _ in range(n): + x += FROZEN_DICT_CONST['x'] + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_INSERT_2_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_BINARY_OP_SUBSCR_DICT", uops) + + def test_binary_subscr_frozendict_const_fold(self): + def testfunc(n): + x = 0 + for _ in range(n): + if FROZEN_DICT_CONST['x'] == 1: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + # lookup result is folded to constant 1, so comparison is optimized away + self.assertNotIn("_COMPARE_OP_INT", uops) + + def test_contains_op_frozenset_const_fold(self): + def testfunc(n): + x = 0 + for _ in range(n): + if 1 in FROZEN_SET_CONST: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_CONTAINS_OP_SET", uops) + def test_binary_subscr_list_slice(self): def testfunc(n): x = 0 + l = [1, 2, 3] for _ in range(n): - l = [1, 2, 3] x += l[0:1][0] return x @@ -3778,7 +4431,7 @@ def testfunc(n): self.assertIn("_BINARY_OP_SUBSCR_LIST_SLICE", uops) self.assertNotIn("_GUARD_TOS_LIST", uops) - self.assertEqual(count_ops(ex, "_POP_TOP"), 3) + self.assertEqual(count_ops(ex, "_POP_TOP"), 2) self.assertEqual(count_ops(ex, "_POP_TOP_NOP"), 4) def test_is_op(self): @@ -4031,6 +4684,21 @@ class A: self.assertIn("_MATCH_CLASS", uops) self.assertEqual(count_ops(ex, "_POP_TOP_NOP"), 4) + def test_dict_update(self): + def testfunc(n): + d = {1: 2, 3: 4} + for _ in range(n): + x = {**d} + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, {1: 2, 3: 4}) + uops = get_opnames(ex) + + self.assertIn("_DICT_UPDATE", uops) + self.assertEqual(count_ops(ex, "_POP_TOP_NOP"), 1) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) + def test_set_update(self): def testfunc(n): s = {1, 2, 3} @@ -4046,6 +4714,38 @@ def testfunc(n): self.assertEqual(count_ops(ex, "_POP_TOP_NOP"), 1) self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) + def test_dict_merge(self): + def testfunc(n): + d = {"a": 1, "b": 2} + def f(**kwargs): + return kwargs + for _ in range(n): + x = f(**d) + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, {"a": 1, "b": 2}) + uops = get_opnames(ex) + + self.assertIn("_DICT_MERGE", uops) + self.assertGreaterEqual(count_ops(ex, "_POP_TOP_NOP"), 1) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) + + def test_list_extend(self): + def testfunc(n): + a = [1, 2, 3] + for _ in range(n): + x = [*a] + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, [1, 2, 3]) + uops = get_opnames(ex) + + self.assertIn("_LIST_EXTEND", uops) + self.assertGreaterEqual(count_ops(ex, "_POP_TOP_NOP"), 1) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) + def test_143026(self): # https://github.com/python/cpython/issues/143026 diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py index a52c2241f5d9d4..1958c44e2b64ef 100644 --- a/Lib/test/test_cext/__init__.py +++ b/Lib/test/test_cext/__init__.py @@ -38,15 +38,15 @@ def test_build(self): self.check_build('_test_cext') def check_build(self, extension_name, std=None, limited=False, - opaque_pyobject=False): + abi3t=False): venv_dir = 'env' with support.setup_venv_with_pip_setuptools(venv_dir) as python_exe: self._check_build(extension_name, python_exe, std=std, limited=limited, - opaque_pyobject=opaque_pyobject) + abi3t=abi3t) def _check_build(self, extension_name, python_exe, std, limited, - opaque_pyobject): + abi3t): pkg_dir = 'pkg' os.mkdir(pkg_dir) shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) @@ -60,8 +60,8 @@ def run_cmd(operation, cmd): env['CPYTHON_TEST_STD'] = std if limited: env['CPYTHON_TEST_LIMITED'] = '1' - if opaque_pyobject: - env['CPYTHON_TEST_OPAQUE_PYOBJECT'] = '1' + if abi3t: + env['CPYTHON_TEST_ABI3T'] = '1' env['CPYTHON_TEST_EXT_NAME'] = extension_name env['TEST_INTERNAL_C_API'] = str(int(self.TEST_INTERNAL_C_API)) if support.verbose: @@ -116,10 +116,9 @@ def test_build_limited_c11(self): def test_build_c11(self): self.check_build('_test_c11_cext', std='c11') - def test_build_opaque_pyobject(self): - # Test with _Py_OPAQUE_PYOBJECT - self.check_build('_test_limited_opaque_cext', limited=True, - opaque_pyobject=True) + def test_build_abi3t(self): + # Test with Py_TARGET_ABI3T + self.check_build('_test_abi3t', abi3t=True) @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99") def test_build_c99(self): diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index 7555b78f18c2e6..a880cb82811f78 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -119,8 +119,10 @@ _Py_COMP_DIAG_PUSH #endif PyDoc_STRVAR(_testcext_doc, "C test extension."); +PyABIInfo_VAR(abi_info); static PyModuleDef_Slot _testcext_slots[] = { + {Py_mod_abi, &abi_info}, {Py_mod_name, STR(MODULE_NAME)}, {Py_mod_doc, (void*)(char*)_testcext_doc}, {Py_mod_exec, (void*)_testcext_exec}, diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index db43f6fb17a132..7262a110d83415 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -59,7 +59,7 @@ def main(): std = os.environ.get("CPYTHON_TEST_STD", "") module_name = os.environ["CPYTHON_TEST_EXT_NAME"] limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", "")) - opaque_pyobject = bool(os.environ.get("CPYTHON_TEST_OPAQUE_PYOBJECT", "")) + abi3t = bool(os.environ.get("CPYTHON_TEST_ABI3T", "")) internal = bool(int(os.environ.get("TEST_INTERNAL_C_API", "0"))) sources = [SOURCE] @@ -91,14 +91,12 @@ def main(): # CC env var overrides sysconfig CC variable in setuptools os.environ['CC'] = cmd - # Define Py_LIMITED_API macro + # Define opt-in macros if limited: - version = sys.hexversion - cflags.append(f'-DPy_LIMITED_API={version:#x}') + cflags.append(f'-DPy_LIMITED_API={sys.hexversion:#x}') - # Define _Py_OPAQUE_PYOBJECT macro - if opaque_pyobject: - cflags.append(f'-D_Py_OPAQUE_PYOBJECT') + if abi3t: + cflags.append(f'-DPy_TARGET_ABI3T={sys.hexversion:#x}') if internal: cflags.append('-DTEST_INTERNAL_C_API=1') diff --git a/Lib/test/test_cmath.py b/Lib/test/test_cmath.py index a96a5780b31b6f..389a3fa0e0a1eb 100644 --- a/Lib/test/test_cmath.py +++ b/Lib/test/test_cmath.py @@ -406,6 +406,8 @@ def polar_with_errno_set(z): _testcapi.set_errno(0) self.check_polar(polar_with_errno_set) + @unittest.skipIf(sys.platform.startswith("sunos"), + "skipping, see gh-138573") def test_phase(self): self.assertAlmostEqual(phase(0), 0.) self.assertAlmostEqual(phase(1.), 0.) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 19fa387cd7b271..fac7e9148f1502 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -1164,6 +1164,18 @@ def test_stateless(self): with self.assertRaises(Exception): _testinternalcapi.verify_stateless_code(func) + def test_code_richcompare_raise_exception(self): + class BadStr(str): + def __eq__(self, _): + raise RuntimeError("Poison!") + + __hash__ = str.__hash__ + + c1 = compile("pass", "test", "exec") + c2 = c1.replace(co_name=BadStr("poison")) + c3 = compile("pass", "poison", "exec") + with self.assertRaises(RuntimeError): + c2 == c3 def isinterned(s): return s is sys.intern(('_' + s + '_')[1:-1]) diff --git a/Lib/test/test_datetime.py b/Lib/test/test_datetime.py index 005187f13e665f..137c8d2686c224 100644 --- a/Lib/test/test_datetime.py +++ b/Lib/test/test_datetime.py @@ -18,16 +18,19 @@ def load_tests(loader, tests, pattern): finally: # XXX: import_fresh_module() is supposed to leave sys.module cache untouched, # XXX: but it does not, so we have to cleanup ourselves. - for modname in ['datetime', '_datetime', '_strptime']: + for modname in ['datetime', '_datetime', '_pydatetime', '_strptime']: sys.modules.pop(modname, None) test_modules = [pure_tests, fast_tests] test_suffixes = ["_Pure", "_Fast"] + # XXX(gb) First run all the _Pure tests, then all the _Fast tests. You might # not believe this, but in spite of all the sys.modules trickery running a _Pure # test last will leave a mix of pure and native datetime stuff lying around. for module, suffix in zip(test_modules, test_suffixes): test_classes = [] + if module is None: + continue for name, cls in module.__dict__.items(): if not isinstance(cls, type): continue @@ -48,8 +51,8 @@ def setUpClass(cls_, module=module): cls_._save_sys_modules = sys.modules.copy() sys.modules[TESTS] = module sys.modules['datetime'] = module.datetime_module - if hasattr(module, '_pydatetime'): - sys.modules['_pydatetime'] = module._pydatetime + sys.modules['_pydatetime'] = module._pydatetime + sys.modules['_datetime'] = module._datetime sys.modules['_strptime'] = module._strptime super().setUpClass() diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index d6e3719479a214..8a8e70214e27ae 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -1803,6 +1803,28 @@ class SubSpam(spam.spamlist): pass spam_cm.__get__(None, list) self.assertEqual(str(cm.exception), expected_errmsg) + @support.cpython_only + def test_method_get_meth_method_invalid_type(self): + # gh-146615: method_get() for METH_METHOD descriptors used to pass + # Py_TYPE(type)->tp_name as the %V fallback instead of the separate + # %s argument, causing a missing argument for %s and a crash. + # Verify the error message is correct when __get__() is called with a + # non-type as the second argument. + # + # METH_METHOD|METH_FASTCALL|METH_KEYWORDS is the only flag combination + # that enters the affected branch in method_get(). + import io + + obj = io.StringIO() + descr = io.TextIOBase.read + + with self.assertRaises(TypeError) as cm: + descr.__get__(obj, "not_a_type") + self.assertEqual( + str(cm.exception), + "descriptor 'read' needs a type, not 'str', as arg 2", + ) + def test_staticmethods(self): # Testing static methods... class C(object): @@ -5361,6 +5383,31 @@ def foo(self): with self.assertRaisesRegex(NotImplementedError, "BAR"): B().foo + def test_gh146587(self): + # See https://github.com/python/cpython/issues/146587 + + class A: + def __radd__(self, other): ... + + class B(tuple): ... + + self.assertIsNone(() + A()) + self.assertIsNone(B() + A()) + + from typing import NamedTuple + + class T(NamedTuple): + x: int + + class A: + def __init__(self, *args): + self.lst = list(args) + def __radd__(self, other): + return A(*self.lst, other) + + self.assertEqual(((1,)+A()).lst, [(1,)]) + self.assertEqual((T(x=1)+A()).lst, [T(x=1)]) + class DictProxyTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index f4210db5bd788e..f8bbcd35ca7c09 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -174,7 +174,7 @@ def bug708901(): %3d LOAD_SMALL_INT 10 %3d CALL 2 - GET_ITER + GET_ITER 0 L1: FOR_ITER 3 (to L2) STORE_FAST 0 (res) @@ -616,6 +616,7 @@ async def _asyncwith(c): LOAD_SPECIAL 2 (__aenter__) CALL 0 GET_AWAITABLE 1 + PUSH_NULL LOAD_CONST 0 (None) L2: SEND 4 (to L5) L3: YIELD_VALUE 1 @@ -632,6 +633,7 @@ async def _asyncwith(c): LOAD_CONST 0 (None) CALL 3 GET_AWAITABLE 2 + PUSH_NULL LOAD_CONST 0 (None) L8: SEND 4 (to L11) L9: YIELD_VALUE 1 @@ -646,12 +648,13 @@ async def _asyncwith(c): RETURN_VALUE %4d L12: CLEANUP_THROW - L13: JUMP_BACKWARD_NO_INTERRUPT 27 (to L5) + L13: JUMP_BACKWARD_NO_INTERRUPT 28 (to L5) L14: CLEANUP_THROW L15: JUMP_BACKWARD_NO_INTERRUPT 10 (to L11) L16: PUSH_EXC_INFO WITH_EXCEPT_START GET_AWAITABLE 2 + PUSH_NULL LOAD_CONST 0 (None) L17: SEND 5 (to L21) L18: YIELD_VALUE 1 @@ -681,15 +684,15 @@ async def _asyncwith(c): RERAISE 1 ExceptionTable: L1 to L3 -> L27 [0] lasti - L3 to L4 -> L12 [4] + L3 to L4 -> L12 [5] L4 to L6 -> L27 [0] lasti L6 to L7 -> L16 [2] lasti L7 to L9 -> L27 [0] lasti - L9 to L10 -> L14 [2] + L9 to L10 -> L14 [3] L10 to L13 -> L27 [0] lasti L14 to L15 -> L27 [0] lasti L16 to L18 -> L26 [4] lasti - L18 to L19 -> L20 [7] + L18 to L19 -> L20 [8] L19 to L22 -> L26 [4] lasti L23 to L25 -> L26 [4] lasti L25 to L27 -> L27 [0] lasti @@ -876,7 +879,7 @@ def foo(x): -- COPY_FREE_VARS 1 %4d LOAD_FAST 0 (.0) - GET_ITER + GET_ITER 0 RETURN_GENERATOR POP_TOP L1: RESUME 0 @@ -933,7 +936,7 @@ def loop_test(): LIST_EXTEND 1 LOAD_SMALL_INT 3 BINARY_OP 5 (*) - GET_ITER + GET_ITER 0 L1: FOR_ITER_LIST 14 (to L2) STORE_FAST 0 (i) @@ -1035,6 +1038,7 @@ def test_widths(self): long_opcodes = set(['JUMP_BACKWARD_NO_INTERRUPT', 'LOAD_FAST_BORROW_LOAD_FAST_BORROW', 'INSTRUMENTED_CALL_FUNCTION_EX', + 'YIELD_FROM_CORO_CHECK', 'ANNOTATIONS_PLACEHOLDER']) for op, opname in enumerate(dis.opname): if opname in long_opcodes or opname.startswith("INSTRUMENTED"): @@ -1855,7 +1859,7 @@ def _prepare_test_cases(): make_inst(opname='LOAD_GLOBAL', arg=1, argval='range', argrepr='range + NULL', offset=4, start_offset=4, starts_line=True, line_number=3, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_SMALL_INT', arg=10, argval=10, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3), make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=16, start_offset=16, starts_line=False, line_number=3, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='GET_ITER', arg=None, argval=None, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=3), + make_inst(opname='GET_ITER', arg=0, argval=0, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=3), make_inst(opname='FOR_ITER', arg=33, argval=96, argrepr='to L4', offset=26, start_offset=26, starts_line=False, line_number=3, label=1, cache_info=[('counter', 1, b'\x00\x00')]), make_inst(opname='STORE_FAST', arg=0, argval='i', argrepr='i', offset=30, start_offset=30, starts_line=False, line_number=3), make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=32, start_offset=32, starts_line=True, line_number=4, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index c62b340f6a340f..20a56ed715d8ab 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -7,6 +7,7 @@ import unittest import unittest.mock from pathlib import Path +from test.support import import_helper import ensurepip import ensurepip._uninstall @@ -36,6 +37,15 @@ def test_version_no_dir(self): # when the bundled pip wheel is used, we get _PIP_VERSION self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version()) + def test_wheel_pkg_dir_none(self): + # gh-146310: empty or None WHEEL_PKG_DIR should not search CWD + for value in ('', None): + with unittest.mock.patch('sysconfig.get_config_var', + return_value=value) as get_config_var: + module = import_helper.import_fresh_module('ensurepip') + self.assertIsNone(module._WHEEL_PKG_DIR) + get_config_var.assert_called_once_with('WHEEL_PKG_DIR') + def test_selected_wheel_path_no_dir(self): pip_filename = f'pip-{ensurepip._PIP_VERSION}-py3-none-any.whl' with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None): diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 7354f8281d9682..3f5fcb29b442de 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -2561,6 +2561,30 @@ def test_incorrect_constructor(self): args = ("bad.py", 1, 2, "abcdefg", 1) self.assertRaises(TypeError, SyntaxError, "bad bad", args) + def test_syntax_error_memory_leak(self): + # gh-146250: memory leak with re-initialization of SyntaxError + e = SyntaxError("msg", ("file.py", 1, 2, "txt", 2, 3)) + e.__init__("new_msg", ("new_file.py", 2, 3, "new_txt", 3, 4)) + self.assertEqual(e.msg, "new_msg") + self.assertEqual(e.args, ("new_msg", ("new_file.py", 2, 3, "new_txt", 3, 4))) + self.assertEqual(e.filename, "new_file.py") + self.assertEqual(e.lineno, 2) + self.assertEqual(e.offset, 3) + self.assertEqual(e.text, "new_txt") + self.assertEqual(e.end_lineno, 3) + self.assertEqual(e.end_offset, 4) + + e = SyntaxError("msg", ("file.py", 1, 2, "txt", 2, 3)) + e.__init__("new_msg", ("new_file.py", 2, 3, "new_txt")) + self.assertEqual(e.msg, "new_msg") + self.assertEqual(e.args, ("new_msg", ("new_file.py", 2, 3, "new_txt"))) + self.assertEqual(e.filename, "new_file.py") + self.assertEqual(e.lineno, 2) + self.assertEqual(e.offset, 3) + self.assertEqual(e.text, "new_txt") + self.assertIsNone(e.end_lineno) + self.assertIsNone(e.end_offset) + class TestInvalidExceptionMatcher(unittest.TestCase): def test_except_star_invalid_exception_type(self): diff --git a/Lib/test/test_extcall.py b/Lib/test/test_extcall.py index f003a5837ae0eb..4da5601e80295f 100644 --- a/Lib/test/test_extcall.py +++ b/Lib/test/test_extcall.py @@ -329,6 +329,22 @@ ... TypeError: Value after ** must be a mapping, not function + >>> class OnlyKeys: + ... def keys(self): + ... return ['key'] + >>> h(**OnlyKeys()) + Traceback (most recent call last): + ... + TypeError: 'OnlyKeys' object is not subscriptable + + >>> class BrokenKeys: + ... def keys(self): + ... return 1 + >>> h(**BrokenKeys()) + Traceback (most recent call last): + ... + TypeError: test.test_extcall.BrokenKeys.keys() must return an iterable, not int + >>> dir(b=1, **{'b': 1}) Traceback (most recent call last): ... @@ -540,6 +556,151 @@ """ +def test_errors_in_iter(): + """ + >>> class A: + ... def __iter__(self): + ... raise exc + ... + >>> def f(*args, **kwargs): pass + >>> exc = ZeroDivisionError('some error') + >>> f(*A()) + Traceback (most recent call last): + ... + ZeroDivisionError: some error + + >>> exc = AttributeError('some error') + >>> f(*A()) + Traceback (most recent call last): + ... + AttributeError: some error + + >>> exc = TypeError('some error') + >>> f(*A()) + Traceback (most recent call last): + ... + TypeError: some error + """ + +def test_errors_in_next(): + """ + >>> class I: + ... def __iter__(self): + ... return self + ... def __next__(self): + ... raise exc + ... + >>> class A: + ... def __iter__(self): + ... return I() + ... + + >>> def f(*args, **kwargs): pass + >>> exc = ZeroDivisionError('some error') + >>> f(*A()) + Traceback (most recent call last): + ... + ZeroDivisionError: some error + + >>> exc = AttributeError('some error') + >>> f(*A()) + Traceback (most recent call last): + ... + AttributeError: some error + + >>> exc = TypeError('some error') + >>> f(*A()) + Traceback (most recent call last): + ... + TypeError: some error + """ + +def test_errors_in_keys(): + """ + >>> class D: + ... def keys(self): + ... raise exc + ... + >>> def f(*args, **kwargs): pass + >>> exc = ZeroDivisionError('some error') + >>> f(**D()) + Traceback (most recent call last): + ... + ZeroDivisionError: some error + + >>> exc = AttributeError('some error') + >>> f(**D()) + Traceback (most recent call last): + ... + AttributeError: some error + + >>> exc = KeyError('some error') + >>> f(**D()) + Traceback (most recent call last): + ... + KeyError: 'some error' + """ + +def test_errors_in_keys_next(): + """ + >>> class I: + ... def __iter__(self): + ... return self + ... def __next__(self): + ... raise exc + ... + >>> class D: + ... def keys(self): + ... return I() + ... + >>> def f(*args, **kwargs): pass + >>> exc = ZeroDivisionError('some error') + >>> f(**D()) + Traceback (most recent call last): + ... + ZeroDivisionError: some error + + >>> exc = AttributeError('some error') + >>> f(**D()) + Traceback (most recent call last): + ... + AttributeError: some error + + >>> exc = KeyError('some error') + >>> f(**D()) + Traceback (most recent call last): + ... + KeyError: 'some error' + """ + +def test_errors_in_getitem(): + """ + >>> class D: + ... def keys(self): + ... return ['key'] + ... def __getitem__(self, key): + ... raise exc + ... + >>> def f(*args, **kwargs): pass + >>> exc = ZeroDivisionError('some error') + >>> f(**D()) + Traceback (most recent call last): + ... + ZeroDivisionError: some error + + >>> exc = AttributeError('some error') + >>> f(**D()) + Traceback (most recent call last): + ... + AttributeError: some error + + >>> exc = KeyError('some error') + >>> f(**D()) + Traceback (most recent call last): + ... + KeyError: 'some error' + """ + import doctest import unittest diff --git a/Lib/test/test_frame_pointer_unwind.py b/Lib/test/test_frame_pointer_unwind.py index 5804cc7e1d7f12..c70ec281686715 100644 --- a/Lib/test/test_frame_pointer_unwind.py +++ b/Lib/test/test_frame_pointer_unwind.py @@ -25,13 +25,19 @@ def _frame_pointers_expected(machine): ) if value ) + if "no-omit-frame-pointer" in cflags: + # For example, configure adds -fno-omit-frame-pointer if Python + # has perf trampoline (PY_HAVE_PERF_TRAMPOLINE) and Python is built + # in debug mode. return True if "omit-frame-pointer" in cflags: return False + if sys.platform == "darwin": # macOS x86_64/ARM64 always have frame pointer by default. return True + if sys.platform == "linux": if machine in {"aarch64", "arm64"}: # 32-bit Linux is not supported @@ -39,7 +45,21 @@ def _frame_pointers_expected(machine): return None return True if machine == "x86_64": + final_opt = "" + for opt in cflags.split(): + if opt.startswith('-O'): + final_opt = opt + if final_opt in ("-O0", "-Og", "-O1"): + # Unwinding works if the optimization level is low + return True + + Py_ENABLE_SHARED = int(sysconfig.get_config_var('Py_ENABLE_SHARED') or '0') + if Py_ENABLE_SHARED: + # Unwinding does crash using gcc -O2 or gcc -O3 + # when Python is built with --enable-shared + return "crash" return False + if sys.platform == "win32": # MSVC ignores /Oy and /Oy- on x64/ARM64. if machine == "arm64": @@ -153,10 +173,14 @@ class FramePointerUnwindTests(unittest.TestCase): def setUp(self): super().setUp() + machine = platform.machine().lower() expected = _frame_pointers_expected(machine) if expected is None: self.skipTest(f"unsupported architecture for frame pointer check: {machine}") + if expected == "crash": + self.skipTest(f"test does crash on {machine}") + try: _testinternalcapi.manual_frame_pointer_unwind() except RuntimeError as exc: diff --git a/Lib/test/test_free_threading/test_itertools.py b/Lib/test/test_free_threading/test_itertools.py index 20135dd3165acf..670d4ca8835e0d 100644 --- a/Lib/test/test_free_threading/test_itertools.py +++ b/Lib/test/test_free_threading/test_itertools.py @@ -1,5 +1,5 @@ import unittest -from itertools import accumulate, batched, chain, combinations_with_replacement, cycle, permutations +from itertools import accumulate, batched, chain, combinations_with_replacement, cycle, permutations, zip_longest from test.support import threading_helper @@ -62,6 +62,13 @@ def test_permutations(self): it = permutations(tuple(range(4)), 2) threading_helper.run_concurrently(work_iterator, nthreads=6, args=[it]) + @threading_helper.reap_threads + def test_zip_longest(self): + number_of_iterations = 10 + for _ in range(number_of_iterations): + it = zip_longest(list(range(4)), list(range(8)), fillvalue=0) + threading_helper.run_concurrently(work_iterator, nthreads=10, args=[it]) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_getpass.py b/Lib/test/test_getpass.py index 9c3def2c3be59b..272414a6204856 100644 --- a/Lib/test/test_getpass.py +++ b/Lib/test/test_getpass.py @@ -88,6 +88,147 @@ def test_trims_trailing_newline(self): input = StringIO('test\n') self.assertEqual('test', getpass._raw_input(input=input)) + def check_raw_input(self, inputs, expect_result, prompt='Password: '): + mock_input = StringIO(inputs) + mock_output = StringIO() + result = getpass._raw_input(prompt, mock_output, mock_input, '*') + self.assertEqual(result, expect_result) + return mock_output.getvalue() + + def test_null_char(self): + self.check_raw_input('pass\x00word\n', 'password') + + def test_raw_input_with_echo_char(self): + output = self.check_raw_input('my1pa$$word!\n', 'my1pa$$word!') + self.assertEqual('Password: ************', output) + + def test_control_chars_with_echo_char(self): + output = self.check_raw_input('pass\twd\b\n', 'pass\tw') + # After backspace: refresh rewrites prompt + 6 echo chars + self.assertEqual( + 'Password: *******' # initial prompt + 7 echo chars + '\r' + ' ' * 17 + '\r' # clear line (10 prompt + 7 prev) + 'Password: ******', # rewrite prompt + 6 echo chars + output + ) + + def test_kill_ctrl_u_with_echo_char(self): + # Ctrl+U (KILL) should clear the entire line + output = self.check_raw_input('foo\x15bar\n', 'bar') + # Should show "***" then refresh to clear, then show "***" for "bar" + self.assertIn('***', output) + # Display refresh uses \r to rewrite the line including prompt + self.assertIn('\r', output) + + def test_werase_ctrl_w_with_echo_char(self): + # Ctrl+W (WERASE) should delete the previous word + self.check_raw_input('hello world\x17end\n', 'hello end') + + def test_ctrl_w_display_preserves_prompt(self): + # Reproducer from gh-138577: type "hello world", Ctrl+W + # Display must show "Password: ******" not "******rd: ***********" + output = self.check_raw_input('hello world\x17\n', 'hello ') + # The final visible state should be "Password: ******" + # Verify prompt is rewritten during refresh, not overwritten by stars + self.assertEndsWith(output, 'Password: ******') + + def test_ctrl_a_insert_display_preserves_prompt(self): + # Reproducer from gh-138577: type "abc", Ctrl+A, type "x" + # Display must show "Password: ****" not "****word: ***" + output = self.check_raw_input('abc\x01x\n', 'xabc') + # The final visible state should be "Password: ****" + self.assertEndsWith(output, 'Password: ****\x08\x08\x08') + + def test_lnext_ctrl_v_with_echo_char(self): + # Ctrl+V (LNEXT) should insert the next character literally + self.check_raw_input('test\x16\x15more\n', 'test\x15more') + + def test_ctrl_a_move_to_start_with_echo_char(self): + # Ctrl+A should move cursor to start + self.check_raw_input('end\x01start\n', 'startend') + + def test_ctrl_a_cursor_position(self): + # After Ctrl+A, cursor is at position 0. + # Refresh writes backspaces to move cursor from end to start. + output = self.check_raw_input('abc\x01\n', 'abc') + self.assertEndsWith(output, 'Password: ***\x08\x08\x08') + + def test_ctrl_a_on_empty(self): + # Ctrl+A on empty line should be a no-op + self.check_raw_input('\x01hello\n', 'hello') + + def test_ctrl_a_already_at_start(self): + # Double Ctrl+A should be same as single Ctrl+A + self.check_raw_input('abc\x01\x01start\n', 'startabc') + + def test_ctrl_a_then_backspace(self): + # Backspace after Ctrl+A should do nothing (cursor at 0) + self.check_raw_input('abc\x01\x7f\n', 'abc') + + def test_ctrl_e_move_to_end_with_echo_char(self): + # Ctrl+E should move cursor to end + self.check_raw_input('start\x01X\x05end\n', 'Xstartend') + + def test_ctrl_e_cursor_position(self): + # After Ctrl+A then Ctrl+E, cursor is back at end. + # Refresh has no backspaces since cursor is at end. + output = self.check_raw_input('abc\x01\x05\n', 'abc') + self.assertEndsWith(output, 'Password: ***') + + def test_ctrl_e_on_empty(self): + # Ctrl+E on empty line should be a no-op + self.check_raw_input('\x05hello\n', 'hello') + + def test_ctrl_e_already_at_end(self): + # Ctrl+E when already at end should be a no-op + self.check_raw_input('abc\x05more\n', 'abcmore') + + def test_ctrl_a_then_ctrl_e(self): + # Ctrl+A then Ctrl+E should return cursor to end, typing appends + self.check_raw_input('abc\x01\x05def\n', 'abcdef') + + def test_ctrl_k_kill_forward_with_echo_char(self): + # Ctrl+K should kill from cursor to end + self.check_raw_input('delete\x01\x0bkeep\n', 'keep') + + def test_ctrl_c_interrupt_with_echo_char(self): + # Ctrl+C should raise KeyboardInterrupt + with self.assertRaises(KeyboardInterrupt): + self.check_raw_input('test\x03more', '') + + def test_ctrl_d_eof_with_echo_char(self): + # Ctrl+D twice should cause EOF + self.check_raw_input('test\x04\x04', 'test') + + def test_backspace_at_start_with_echo_char(self): + # Backspace at start should do nothing + self.check_raw_input('\x7fhello\n', 'hello') + + def test_ctrl_k_at_end_with_echo_char(self): + # Ctrl+K at end should do nothing + self.check_raw_input('hello\x0b\n', 'hello') + + def test_ctrl_w_on_empty_with_echo_char(self): + # Ctrl+W on empty line should do nothing + self.check_raw_input('\x17hello\n', 'hello') + + def test_ctrl_u_on_empty_with_echo_char(self): + # Ctrl+U on empty line should do nothing + self.check_raw_input('\x15hello\n', 'hello') + + def test_multiple_ctrl_operations_with_echo_char(self): + # Test combination: type, move, insert, delete + # "world", Ctrl+A, "hello ", Ctrl+E, "!", Ctrl+A, Ctrl+K, "start" + self.check_raw_input('world\x01hello \x05!\x01\x0bstart\n', 'start') + + def test_ctrl_w_multiple_words_with_echo_char(self): + # Ctrl+W should delete only the last word + self.check_raw_input('one two three\x17\n', 'one two ') + + def test_ctrl_v_then_ctrl_c_with_echo_char(self): + # Ctrl+V should make Ctrl+C literal (not interrupt) + self.check_raw_input('test\x16\x03end\n', 'test\x03end') + # Some of these tests are a bit white-box. The functional requirement is that # the password input be taken directly from the tty, and that it not be echoed @@ -174,33 +315,10 @@ def test_echo_char_replaces_input_with_asterisks(self): result = getpass.unix_getpass(echo_char='*') mock_input.assert_called_once_with('Password: ', textio(), - input=textio(), echo_char='*') + input=textio(), echo_char='*', + term_ctrl_chars=mock.ANY) self.assertEqual(result, mock_result) - def test_raw_input_with_echo_char(self): - passwd = 'my1pa$$word!' - mock_input = StringIO(f'{passwd}\n') - mock_output = StringIO() - with mock.patch('sys.stdin', mock_input), \ - mock.patch('sys.stdout', mock_output): - result = getpass._raw_input('Password: ', mock_output, mock_input, - '*') - self.assertEqual(result, passwd) - self.assertEqual('Password: ************', mock_output.getvalue()) - - def test_control_chars_with_echo_char(self): - passwd = 'pass\twd\b' - expect_result = 'pass\tw' - mock_input = StringIO(f'{passwd}\n') - mock_output = StringIO() - with mock.patch('sys.stdin', mock_input), \ - mock.patch('sys.stdout', mock_output): - result = getpass._raw_input('Password: ', mock_output, mock_input, - '*') - self.assertEqual(result, expect_result) - self.assertEqual('Password: *******\x08 \x08', mock_output.getvalue()) - - class GetpassEchoCharTest(unittest.TestCase): def test_accept_none(self): diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 437ab7031356b1..c905c0da0a1232 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -3499,12 +3499,20 @@ class Sub(tp): pass self.assertEqual(_testcapi.pytype_getmodulebytoken(Sub, token), module) - @requires_gil_enabled("empty slots re-enable GIL") def test_from_modexport_empty_slots(self): + # Module to test that Py_mod_abi is mandatory for PyModExport + modname = '_test_from_modexport_empty_slots' + filename = _testmultiphase.__file__ + with self.assertRaises(SystemError): + import_extension_from_file( + modname, filename, put_in_sys_modules=False) + + @requires_gil_enabled("this module re-enables GIL") + def test_from_modexport_minimal_slots(self): # Module to test that: - # - no slots are mandatory for PyModExport + # - no slots except Py_mod_abi is mandatory for PyModExport # - the slots array is used as the default token - modname = '_test_from_modexport_empty_slots' + modname = '_test_from_modexport_minimal_slots' filename = _testmultiphase.__file__ module = import_extension_from_file( modname, filename, put_in_sys_modules=False) @@ -3516,7 +3524,7 @@ def test_from_modexport_empty_slots(self): smoke_mod = import_extension_from_file( '_test_from_modexport_smoke', filename, put_in_sys_modules=False) self.assertEqual(_testcapi.pymodule_get_token(module), - smoke_mod.get_modexport_empty_slots()) + smoke_mod.get_modexport_minimal_slots()) @cpython_only class TestMagicNumber(unittest.TestCase): diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py index cdc8884d668a66..dc77fa78a203fd 100644 --- a/Lib/test/test_importlib/extension/test_finder.py +++ b/Lib/test/test_importlib/extension/test_finder.py @@ -1,4 +1,4 @@ -from test.support import is_apple_mobile +from test.support import is_apple_mobile, Py_GIL_DISABLED from test.test_importlib import abc, util machinery = util.import_importlib('importlib.machinery') @@ -59,6 +59,20 @@ def test_module(self): def test_failure(self): self.assertIsNone(self.find_spec('asdfjkl;')) + def test_abi3_extension_suffixes(self): + suffixes = self.machinery.EXTENSION_SUFFIXES + if 'win32' in sys.platform: + # Either "_d.pyd" or ".pyd" must be in suffixes + self.assertTrue({"_d.pyd", ".pyd"}.intersection(suffixes)) + elif 'cygwin' in sys.platform: + pass + else: + if Py_GIL_DISABLED: + self.assertNotIn(".abi3.so", suffixes) + else: + self.assertIn(".abi3.so", suffixes) + self.assertIn(".abi3t.so", suffixes) + (Frozen_FinderTests, Source_FinderTests diff --git a/Lib/test/test_importlib/metadata/fixtures.py b/Lib/test/test_importlib/metadata/fixtures.py index ad0ab42e089a9d..3283697d418188 100644 --- a/Lib/test/test_importlib/metadata/fixtures.py +++ b/Lib/test/test_importlib/metadata/fixtures.py @@ -6,6 +6,7 @@ import shutil import sys import textwrap +from importlib import resources from test.support import import_helper from test.support import os_helper @@ -14,11 +15,6 @@ from . import _path from ._path import FilesSpec -if sys.version_info >= (3, 9): - from importlib import resources -else: - import importlib_resources as resources - @contextlib.contextmanager def tmp_path(): @@ -374,8 +370,6 @@ def setUp(self): # Add self.zip_name to the front of sys.path. self.resources = contextlib.ExitStack() self.addCleanup(self.resources.close) - # workaround for #138313 - self.addCleanup(lambda: None) def parameterize(*args_set): diff --git a/Lib/test/test_importlib/metadata/test_api.py b/Lib/test/test_importlib/metadata/test_api.py index 3c856a88b77bf6..5449f0484492fb 100644 --- a/Lib/test/test_importlib/metadata/test_api.py +++ b/Lib/test/test_importlib/metadata/test_api.py @@ -317,33 +317,31 @@ def test_invalidate_cache(self): class PreparedTests(unittest.TestCase): - def test_normalize(self): - tests = [ - # Simple - ("sample", "sample"), - # Mixed case - ("Sample", "sample"), - ("SAMPLE", "sample"), - ("SaMpLe", "sample"), - # Separator conversions - ("sample-pkg", "sample_pkg"), - ("sample.pkg", "sample_pkg"), - ("sample_pkg", "sample_pkg"), - # Multiple separators - ("sample---pkg", "sample_pkg"), - ("sample___pkg", "sample_pkg"), - ("sample...pkg", "sample_pkg"), - # Mixed separators - ("sample-._pkg", "sample_pkg"), - ("sample_.-pkg", "sample_pkg"), - # Complex - ("Sample__Pkg-name.foo", "sample_pkg_name_foo"), - ("Sample__Pkg.name__foo", "sample_pkg_name_foo"), - # Uppercase with separators - ("SAMPLE-PKG", "sample_pkg"), - ("Sample.Pkg", "sample_pkg"), - ("SAMPLE_PKG", "sample_pkg"), - ] - for name, expected in tests: - with self.subTest(name=name): - self.assertEqual(Prepared.normalize(name), expected) + @fixtures.parameterize( + # Simple + dict(input='sample', expected='sample'), + # Mixed case + dict(input='Sample', expected='sample'), + dict(input='SAMPLE', expected='sample'), + dict(input='SaMpLe', expected='sample'), + # Separator conversions + dict(input='sample-pkg', expected='sample_pkg'), + dict(input='sample.pkg', expected='sample_pkg'), + dict(input='sample_pkg', expected='sample_pkg'), + # Multiple separators + dict(input='sample---pkg', expected='sample_pkg'), + dict(input='sample___pkg', expected='sample_pkg'), + dict(input='sample...pkg', expected='sample_pkg'), + # Mixed separators + dict(input='sample-._pkg', expected='sample_pkg'), + dict(input='sample_.-pkg', expected='sample_pkg'), + # Complex + dict(input='Sample__Pkg-name.foo', expected='sample_pkg_name_foo'), + dict(input='Sample__Pkg.name__foo', expected='sample_pkg_name_foo'), + # Uppercase with separators + dict(input='SAMPLE-PKG', expected='sample_pkg'), + dict(input='Sample.Pkg', expected='sample_pkg'), + dict(input='SAMPLE_PKG', expected='sample_pkg'), + ) + def test_normalize(self, input, expected): + self.assertEqual(Prepared.normalize(input), expected) diff --git a/Lib/test/test_importlib/metadata/test_main.py b/Lib/test/test_importlib/metadata/test_main.py index 83b686babfdb7a..aae052160d9763 100644 --- a/Lib/test/test_importlib/metadata/test_main.py +++ b/Lib/test/test_importlib/metadata/test_main.py @@ -2,16 +2,17 @@ import pickle import re import unittest -from test.support import os_helper try: import pyfakefs.fake_filesystem_unittest as ffs except ImportError: from .stubs import fake_filesystem_unittest as ffs +from test.support import os_helper from importlib.metadata import ( Distribution, EntryPoint, + MetadataNotFound, PackageNotFoundError, _unique, distributions, @@ -159,13 +160,15 @@ def test_valid_dists_preferred(self): def test_missing_metadata(self): """ - Dists with a missing metadata file should return None. + Dists with a missing metadata file should raise ``MetadataNotFound``. - Ref python/importlib_metadata#493. + Ref python/importlib_metadata#493 and python/cpython#143387. """ fixtures.build_files(self.make_pkg('foo-4.3', files={}), self.site_dir) - assert Distribution.from_name('foo').metadata is None - assert metadata('foo') is None + with self.assertRaises(MetadataNotFound): + Distribution.from_name('foo').metadata + with self.assertRaises(MetadataNotFound): + metadata('foo') class NonASCIITests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): diff --git a/Lib/test/test_importlib/metadata/test_zip.py b/Lib/test/test_importlib/metadata/test_zip.py index fcb649f3736076..97168549667de3 100644 --- a/Lib/test/test_importlib/metadata/test_zip.py +++ b/Lib/test/test_importlib/metadata/test_zip.py @@ -1,7 +1,12 @@ +import multiprocessing +import os import sys import unittest +from test.support import requires_fork, warnings_helper + from importlib.metadata import ( + FastPath, PackageNotFoundError, distribution, distributions, @@ -47,6 +52,39 @@ def test_one_distribution(self): dists = list(distributions(path=sys.path[:1])) assert len(dists) == 1 + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + @requires_fork() + @unittest.skipUnless( + hasattr(os, 'register_at_fork') + and 'fork' in multiprocessing.get_all_start_methods(), + 'requires fork-based multiprocessing support', + ) + def test_fastpath_cache_cleared_in_forked_child(self): + zip_path = sys.path[0] + + FastPath(zip_path) + assert FastPath.__new__.cache_info().currsize >= 1 + + ctx = multiprocessing.get_context('fork') + parent_conn, child_conn = ctx.Pipe() + + def child(conn, root): + try: + before = FastPath.__new__.cache_info().currsize + FastPath(root) + after = FastPath.__new__.cache_info().currsize + conn.send((before, after)) + finally: + conn.close() + + proc = ctx.Process(target=child, args=(child_conn, zip_path)) + proc.start() + child_conn.close() + cache_sizes = parent_conn.recv() + proc.join() + + self.assertEqual(cache_sizes, (0, 1)) + class TestEgg(TestZip): def setUp(self): diff --git a/Lib/test/test_importlib/test_discover.py b/Lib/test/test_importlib/test_discover.py index 8c5fa65a564c6d..c4ab7b6982e3ba 100644 --- a/Lib/test/test_importlib/test_discover.py +++ b/Lib/test/test_importlib/test_discover.py @@ -11,7 +11,7 @@ def __init__(self, discover=[]): self._discovered_values = discover def find_spec(self, fullname, path=None, target=None): - raise NotImplemented + raise NotImplementedError def discover(self, parent=None): yield from self._discovered_values diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index dc64288085fa74..cf579d4da4e0df 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -754,6 +754,38 @@ def keys(): next(g) next(g) # must pass with address sanitizer + def test_grouper_reentrant_eq_does_not_crash(self): + # regression test for gh-146613 + grouper_iter = None + + class Key: + __hash__ = None + + def __init__(self, do_advance): + self.do_advance = do_advance + + def __eq__(self, other): + nonlocal grouper_iter + if self.do_advance: + self.do_advance = False + if grouper_iter is not None: + try: + next(grouper_iter) + except StopIteration: + pass + return NotImplemented + return True + + def keyfunc(element): + if element == 0: + return Key(do_advance=True) + return Key(do_advance=False) + + g = itertools.groupby(range(4), keyfunc) + key, grouper_iter = next(g) + items = list(grouper_iter) + self.assertEqual(len(items), 1) + def test_filter(self): self.assertEqual(list(filter(isEven, range(6))), [0,2,4]) self.assertEqual(list(filter(None, [0,1,0,2,0])), [1,2]) diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py index 2250af964c022b..d846c8af7ec434 100644 --- a/Lib/test/test_json/test_decode.py +++ b/Lib/test/test_json/test_decode.py @@ -69,6 +69,24 @@ def test_object_pairs_hook(self): object_pairs_hook=OrderedDict), OrderedDict([('empty', OrderedDict())])) + def test_array_hook(self): + s = '[1, 2, 3]' + t = self.loads(s, array_hook=tuple) + self.assertEqual(t, (1, 2, 3)) + self.assertEqual(type(t), tuple) + + # Nested array in inner structure with object_hook + s = '{"xkd": [[1], [2], [3]]}' + p = frozendict(xkd=((1,), (2,), (3,))) + data = self.loads(s, object_hook=frozendict, array_hook=tuple) + self.assertEqual(data, p) + self.assertEqual(type(data), frozendict) + self.assertEqual(type(data["xkd"]), tuple) + for item in data["xkd"]: + self.assertEqual(type(item), tuple) + + self.assertEqual(self.loads('[]', array_hook=tuple), ()) + def test_decoder_optimizations(self): # Several optimizations were made that skip over calls to # the whitespace regex, so this test is designed to try and diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index a4180f05dbbafc..69cb96cf4a0c1a 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -484,8 +484,8 @@ def my_filter(name): sys.set_lazy_imports_filter(my_filter) self.assertIs(sys.get_lazy_imports_filter(), my_filter) - def test_lazy_modules_attribute_is_set(self): - """sys.lazy_modules should be a set per PEP 810.""" + def test_lazy_modules_attribute_is_dict(self): + """sys.lazy_modules should be a dict per PEP 810.""" self.assertIsInstance(sys.lazy_modules, dict) @support.requires_subprocess() @@ -966,9 +966,24 @@ def test_module_added_to_lazy_modules_on_lazy_import(self): def test_lazy_modules_is_per_interpreter(self): """Each interpreter should have independent sys.lazy_modules.""" - # Basic test that sys.lazy_modules exists and is a set + # Basic test that sys.lazy_modules exists and is a dict self.assertIsInstance(sys.lazy_modules, dict) + def test_lazy_module_without_children_is_tracked(self): + code = textwrap.dedent(""" + import sys + lazy import json + assert "json" in sys.lazy_modules, ( + f"expected 'json' in sys.lazy_modules, got {set(sys.lazy_modules)}" + ) + assert sys.lazy_modules["json"] == set(), ( + f"expected empty set for sys.lazy_modules['json'], " + f"got {sys.lazy_modules['json']!r}" + ) + print("OK") + """) + assert_python_ok("-c", code) + @support.requires_subprocess() class CommandLineAndEnvVarTests(unittest.TestCase): @@ -1088,6 +1103,49 @@ def test_env_var_lazy_imports_none_disables_all_lazy(self): self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") self.assertIn("EAGER", result.stdout) + def test_cli_lazy_imports_none_disables_dunder_lazy_modules(self): + """-X lazy_imports=none should override __lazy_modules__.""" + code = textwrap.dedent(""" + import sys + __lazy_modules__ = ["json"] + import json + if 'json' in sys.modules: + print("EAGER") + else: + print("LAZY") + """) + result = subprocess.run( + [sys.executable, "-X", "lazy_imports=none", "-c", code], + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") + self.assertIn("EAGER", result.stdout) + + def test_env_var_lazy_imports_none_disables_dunder_lazy_modules(self): + """PYTHON_LAZY_IMPORTS=none should override __lazy_modules__.""" + code = textwrap.dedent(""" + import sys + __lazy_modules__ = ["json"] + import json + if 'json' in sys.modules: + print("EAGER") + else: + print("LAZY") + """) + import os + + env = os.environ.copy() + env["PYTHON_LAZY_IMPORTS"] = "none" + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + env=env, + ) + self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") + self.assertIn("EAGER", result.stdout) + def test_cli_overrides_env_var(self): """Command-line option should take precedence over environment variable.""" # PEP 810: -X lazy_imports takes precedence over PYTHON_LAZY_IMPORTS @@ -1532,7 +1590,7 @@ def access_modules(idx): self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") self.assertIn("OK", result.stdout) - def test_concurrent_lazy_modules_set_updates(self): + def test_concurrent_lazy_modules_dict_updates(self): """Multiple threads creating lazy imports should safely update sys.lazy_modules.""" code = textwrap.dedent(""" import sys diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 05dcea6ce0e98a..1a76c2173a3011 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -404,6 +404,20 @@ def test_empty_filter(self): r = logging.makeLogRecord({'name': 'spam.eggs'}) self.assertTrue(f.filter(r)) + def test_filter_repr(self): + f = logging.Filter('myapp') + self.assertEqual(repr(f), '') + + def test_filter_repr_empty(self): + f = logging.Filter() + self.assertEqual(repr(f), '') + + def test_filter_repr_subclass(self): + class MyFilter(logging.Filter): + pass + f = MyFilter('myapp') + self.assertEqual(repr(f), '') + # # First, we define our levels. There can be as many as you want - the only # limitations are that they should be integers, the lowest should be > 0 and @@ -4914,6 +4928,20 @@ def test_relativeCreated_has_higher_precision(self): # After PR gh-102412, precision (places) increases from 3 to 7 self.assertAlmostEqual(relativeCreated, offset_ns / 1e6, places=7) + def test_formatter_repr(self): + f = logging.Formatter('%(message)s') + self.assertEqual(repr(f), '') + + def test_formatter_repr_default(self): + f = logging.Formatter() + self.assertEqual(repr(f), '') + + def test_formatter_repr_subclass(self): + class MyFormatter(logging.Formatter): + pass + f = MyFormatter('%(message)s') + self.assertEqual(repr(f), '') + class TestBufferingFormatter(logging.BufferingFormatter): def formatHeader(self, records): diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 7421076ddd4c3a..019c699bff55c4 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -11,6 +11,7 @@ from test.support import import_helper, warnings_helper from test.support import os_helper from test.support import refleak_helper +from test.support import requires_root_user from test.support import socket_helper import unittest import textwrap @@ -1086,6 +1087,7 @@ def test_permissions_after_flush(self): self.assertEqual(os.stat(self._path).st_mode, mode) + @requires_root_user @unittest.skipUnless(hasattr(os, 'chown'), 'requires os.chown') def test_ownership_after_flush(self): # See issue gh-117467 @@ -1108,10 +1110,7 @@ def test_ownership_after_flush(self): else: self.skipTest("test needs more than one group") - try: - os.chown(self._path, other_uid, other_gid) - except OSError: - self.skipTest('test needs root privilege') + os.chown(self._path, other_uid, other_gid) # Change permissions as in test_permissions_after_flush. mode = st.st_mode | 0o666 os.chmod(self._path, mode) diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 68f41a2e62034d..8f9a239bead130 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -291,6 +291,8 @@ def testAtanh(self): self.assertRaises(ValueError, math.atanh, NINF) self.assertTrue(math.isnan(math.atanh(NAN))) + @unittest.skipIf(sys.platform.startswith("sunos"), + "skipping, see gh-138573") def testAtan2(self): self.assertRaises(TypeError, math.atan2) self.ftest('atan2(-1, 0)', math.atan2(-1, 0), -math.pi/2) diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 0f7dc15b8c6f2c..22b9f6af758f88 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -636,6 +636,18 @@ def check_equal(view, is_equal): m = memoryview(a) check_equal(m, True) + # Test complex formats + for complex_format in 'FD': + with self.subTest(format=complex_format): + data = struct.pack(complex_format * 3, 1.0, 2.0, float('nan')) + m = memoryview(data).cast(complex_format) + # nan is not equal to nan + check_equal(m, False) + + data = struct.pack(complex_format * 3, 1.0, 2.0, 3.0) + m = memoryview(data).cast(complex_format) + check_equal(m, True) + class BytesMemorySliceTest(unittest.TestCase, BaseMemorySliceTests, BaseBytesMemoryTests): @@ -682,6 +694,14 @@ def test_half_float(self): self.assertEqual(half_view.nbytes * 2, float_view.nbytes) self.assertListEqual(half_view.tolist(), float_view.tolist()) + def test_complex_types(self): + float_complex_data = struct.pack('FFF', 0.0, -1.5j, 1+2j) + double_complex_data = struct.pack('DDD', 0.0, -1.5j, 1+2j) + float_complex_view = memoryview(float_complex_data).cast('F') + double_complex_view = memoryview(double_complex_data).cast('D') + self.assertEqual(float_complex_view.nbytes * 2, double_complex_view.nbytes) + self.assertListEqual(float_complex_view.tolist(), double_complex_view.tolist()) + def test_memoryview_hex(self): # Issue #9951: memoryview.hex() segfaults with non-contiguous buffers. x = b'0' * 200000 @@ -698,10 +718,10 @@ def test_memoryview_hex_separator(self): self.assertEqual(m2.hex(':', -2), '6564:6362:61') self.assertEqual(m2.hex(sep=':', bytes_per_sep=2), '65:6463:6261') self.assertEqual(m2.hex(sep=':', bytes_per_sep=-2), '6564:6362:61') - for bytes_per_sep in 5, -5, 2**31-1, -(2**31-1): + for bytes_per_sep in 5, -5, sys.maxsize, -sys.maxsize: with self.subTest(bytes_per_sep=bytes_per_sep): self.assertEqual(m2.hex(':', bytes_per_sep), '6564636261') - for bytes_per_sep in 2**31, -2**31, 2**1000, -2**1000: + for bytes_per_sep in sys.maxsize+1, -sys.maxsize-1, 2**1000, -2**1000: with self.subTest(bytes_per_sep=bytes_per_sep): try: self.assertEqual(m2.hex(':', bytes_per_sep), '6564636261') diff --git a/Lib/test/test_netrc.py b/Lib/test/test_netrc.py index 9d720f627102e3..354081e96213a6 100644 --- a/Lib/test/test_netrc.py +++ b/Lib/test/test_netrc.py @@ -1,6 +1,9 @@ import netrc, os, unittest, sys, textwrap +from pathlib import Path from test import support from test.support import os_helper +from unittest.mock import patch + temp_filename = os_helper.TESTFN @@ -309,6 +312,26 @@ def test_security(self): self.assertEqual(nrc.hosts['foo.domain.com'], ('anonymous', '', 'pass')) + @unittest.skipUnless(os.name == 'posix', 'POSIX only test') + @unittest.skipUnless(hasattr(os, 'getuid'), "os.getuid is required") + @os_helper.skip_unless_working_chmod + def test_security_only_once(self): + # Make sure security check is only run once per parse when multiple + # entries are found. + with patch.object(netrc.netrc, "_security_check") as mock: + with os_helper.temp_dir() as tmp_dir: + netrc_path = Path(tmp_dir) / '.netrc' + netrc_path.write_text("""\ + machine foo.domain.com login bar password pass + machine bar.domain.com login foo password pass + """) + netrc_path.chmod(0o600) + with os_helper.EnvironmentVarGuard() as environ: + environ.set('HOME', tmp_dir) + netrc.netrc() + + mock.assert_called_once() + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index 4204a6a47d2a81..642c2722711c7c 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -698,6 +698,7 @@ def test_merge_operator(self): d |= list(b.items()) expected = OrderedDict({0: 0, 1: 1, 2: 2, 3: 3}) self.assertEqual(a | dict(b), expected) + self.assertEqual(a | frozendict(b), expected) self.assertEqual(a | b, expected) self.assertEqual(c, expected) self.assertEqual(d, expected) @@ -706,12 +707,17 @@ def test_merge_operator(self): c |= a expected = OrderedDict({1: 1, 2: 1, 3: 3, 0: 0}) self.assertEqual(dict(b) | a, expected) + self.assertEqual(frozendict(b) | a, expected) + self.assertEqual(a.__ror__(frozendict(b)), expected) self.assertEqual(b | a, expected) self.assertEqual(c, expected) self.assertIs(type(a | b), OrderedDict) self.assertIs(type(dict(a) | b), OrderedDict) + self.assertIs(type(frozendict(a) | b), frozendict) + self.assertIs(type(b.__ror__(frozendict(a))), OrderedDict) self.assertIs(type(a | dict(b)), OrderedDict) + self.assertIs(type(a | frozendict(b)), OrderedDict) expected = a.copy() a |= () diff --git a/Lib/test/test_os/test_os.py b/Lib/test/test_os/test_os.py index 06f69caad12bc8..7e670e5a139d99 100644 --- a/Lib/test/test_os/test_os.py +++ b/Lib/test/test_os/test_os.py @@ -33,6 +33,8 @@ from test.support import os_helper from test.support import socket_helper from test.support import infinite_recursion +from test.support import requires_root_user +from test.support import requires_non_root_user from test.support import warnings_helper from platform import win32_is_iot from .utils import create_file @@ -67,10 +69,6 @@ from test.support.os_helper import FakePath -root_in_posix = False -if hasattr(os, 'geteuid'): - root_in_posix = (os.geteuid() == 0) - # Detect whether we're on a Linux system that uses the (now outdated # and unmaintained) linuxthreads threading library. There's an issue # when combining linuxthreads with a failed execv call: see @@ -2257,8 +2255,8 @@ def test_chown_gid(self): gid = os.stat(os_helper.TESTFN).st_gid self.assertEqual(gid, gid_2) - @unittest.skipUnless(root_in_posix and len(all_users) > 1, - "test needs root privilege and more than one user") + @requires_root_user + @unittest.skipUnless(len(all_users) > 1, "test needs more than one user") def test_chown_with_root(self): uid_1, uid_2 = all_users[:2] gid = os.stat(os_helper.TESTFN).st_gid @@ -2269,8 +2267,8 @@ def test_chown_with_root(self): uid = os.stat(os_helper.TESTFN).st_uid self.assertEqual(uid, uid_2) - @unittest.skipUnless(not root_in_posix and len(all_users) > 1, - "test needs non-root account and more than one user") + @requires_non_root_user + @unittest.skipUnless(len(all_users) > 1, "test needs and more than one user") def test_chown_without_permission(self): uid_1, uid_2 = all_users[:2] gid = os.stat(os_helper.TESTFN).st_gid diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index ef9ea0d11d06a6..19f4506c109c14 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -20,6 +20,8 @@ from test.support import is_emscripten, is_wasi, is_wasm32 from test.support import infinite_recursion from test.support import os_helper +from test.support import requires_root_user +from test.support import requires_non_root_user from test.support.os_helper import TESTFN, FS_NONASCII, FakePath try: import fcntl @@ -35,11 +37,6 @@ posix = None -root_in_posix = False -if hasattr(os, 'geteuid'): - root_in_posix = (os.geteuid() == 0) - - def patch_replace(old_test): def new_replace(self, target): raise OSError(errno.EXDEV, "Cross-device link", self, target) @@ -1554,7 +1551,7 @@ def raiser(*args, **kwargs): self.assertRaises(FileNotFoundError, source.copy, target) @unittest.skipIf(sys.platform == "win32" or sys.platform == "wasi", "directories are always readable on Windows and WASI") - @unittest.skipIf(root_in_posix, "test fails with root privilege") + @requires_non_root_user def test_copy_dir_no_read_permission(self): base = self.cls(self.base) source = base / 'dirE' @@ -2027,7 +2024,7 @@ def test_owner(self): self.assertEqual(expected_name, p.owner()) @unittest.skipUnless(pwd, "the pwd module is needed for this test") - @unittest.skipUnless(root_in_posix, "test needs root privilege") + @requires_root_user def test_owner_no_follow_symlinks(self): all_users = [u.pw_uid for u in pwd.getpwall()] if len(all_users) < 2: @@ -2062,7 +2059,7 @@ def test_group(self): self.assertEqual(expected_name, p.group()) @unittest.skipUnless(grp, "the grp module is needed for this test") - @unittest.skipUnless(root_in_posix, "test needs root privilege") + @requires_root_user def test_group_no_follow_symlinks(self): all_groups = [g.gr_gid for g in grp.getgrall()] if len(all_groups) < 2: diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 88d20bbb028d6f..e0cc010f15513b 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -1470,7 +1470,7 @@ def test_optimize_literal_list_for_iter(self): ('LOAD_SMALL_INT', 1, 0), ('LOAD_SMALL_INT', 2, 0), ('BUILD_LIST', 2, 0), - ('GET_ITER', None, 0), + ('GET_ITER', 0, 0), start := self.Label(), ('FOR_ITER', end := self.Label(), 0), ('STORE_FAST', 0, 0), @@ -1483,7 +1483,7 @@ def test_optimize_literal_list_for_iter(self): ] after = [ ('LOAD_CONST', 1, 0), - ('GET_ITER', None, 0), + ('GET_ITER', 0, 0), start := self.Label(), ('FOR_ITER', end := self.Label(), 0), ('STORE_FAST', 0, 0), @@ -1501,7 +1501,7 @@ def test_optimize_literal_list_for_iter(self): ('LOAD_SMALL_INT', 1, 0), ('LOAD_NAME', 0, 0), ('BUILD_LIST', 2, 0), - ('GET_ITER', None, 0), + ('GET_ITER', 0, 0), start := self.Label(), ('FOR_ITER', end := self.Label(), 0), ('STORE_FAST', 0, 0), @@ -1516,7 +1516,7 @@ def test_optimize_literal_list_for_iter(self): ('LOAD_SMALL_INT', 1, 0), ('LOAD_NAME', 0, 0), ('BUILD_TUPLE', 2, 0), - ('GET_ITER', None, 0), + ('GET_ITER', 0, 0), start := self.Label(), ('FOR_ITER', end := self.Label(), 0), ('STORE_FAST', 0, 0), @@ -1535,7 +1535,7 @@ def test_optimize_literal_set_for_iter(self): ('LOAD_SMALL_INT', 1, 0), ('LOAD_SMALL_INT', 2, 0), ('BUILD_SET', 2, 0), - ('GET_ITER', None, 0), + ('GET_ITER', 0, 0), start := self.Label(), ('FOR_ITER', end := self.Label(), 0), ('STORE_FAST', 0, 0), @@ -1548,7 +1548,7 @@ def test_optimize_literal_set_for_iter(self): ] after = [ ('LOAD_CONST', 1, 0), - ('GET_ITER', None, 0), + ('GET_ITER', 0, 0), start := self.Label(), ('FOR_ITER', end := self.Label(), 0), ('STORE_FAST', 0, 0), @@ -1567,7 +1567,7 @@ def test_optimize_literal_set_for_iter(self): ('LOAD_SMALL_INT', 1, 0), ('LOAD_NAME', 0, 0), ('BUILD_SET', 2, 0), - ('GET_ITER', None, 0), + ('GET_ITER', 0, 0), start := self.Label(), ('FOR_ITER', end := self.Label(), 0), ('STORE_FAST', 0, 0), @@ -2356,7 +2356,7 @@ def test_list_to_tuple_get_iter(self): ("LOAD_FAST", 1, 4), ("LIST_EXTEND", 1, 5), ("CALL_INTRINSIC_1", INTRINSIC_LIST_TO_TUPLE, 6), - ("GET_ITER", None, 7), + ("GET_ITER", 0, 7), top := self.Label(), ("FOR_ITER", end := self.Label(), 8), ("STORE_FAST", 2, 9), @@ -2374,7 +2374,7 @@ def test_list_to_tuple_get_iter(self): ("LOAD_FAST_BORROW", 1, 4), ("LIST_EXTEND", 1, 5), ("NOP", None, 6), # ("CALL_INTRINSIC_1", INTRINSIC_LIST_TO_TUPLE, 6), - ("GET_ITER", None, 7), + ("GET_ITER", 0, 7), top := self.Label(), ("FOR_ITER", end := self.Label(), 8), ("STORE_FAST", 2, 9), @@ -2677,20 +2677,20 @@ def test_set_function_attribute(self): self.cfg_optimization_test(insts, expected, consts=[None]) def test_get_yield_from_iter(self): - # GET_YIELD_FROM_ITER may leave its operand on the stack insts = [ ("LOAD_FAST", 0, 1), - ("GET_YIELD_FROM_ITER", None, 2), - ("LOAD_CONST", 0, 3), + ("GET_ITER", 1, 2), + ("PUSH_NULL", None, 3), + ("LOAD_CONST", 0, 4), send := self.Label(), - ("SEND", end := self.Label(), 5), - ("YIELD_VALUE", 1, 6), - ("RESUME", 2, 7), - ("JUMP", send, 8), + ("SEND", end := self.Label(), 6), + ("YIELD_VALUE", 1, 7), + ("RESUME", 2, 8), + ("JUMP", send, 9), end, - ("END_SEND", None, 9), - ("LOAD_CONST", 0, 10), - ("RETURN_VALUE", None, 11), + ("END_SEND", None, 10), + ("LOAD_CONST", 0, 11), + ("RETURN_VALUE", None, 12), ] self.cfg_optimization_test(insts, insts, consts=[None]) diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index d9216be4d95658..b9c261310bb567 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -792,6 +792,25 @@ def test_dict_members(self): }) self.assertIsNot(pl2['first'], pl2['second']) + def test_frozendict(self): + pl = frozendict( + aString="Doodah", + anInt=728, + aDict=frozendict( + anotherString="hello", + aTrueValue=True, + ), + aList=["A", "B", 12], + ) + + for fmt in ALL_FORMATS: + with self.subTest(fmt=fmt): + data = plistlib.dumps(pl, fmt=fmt) + pl2 = plistlib.loads(data) + self.assertEqual(pl2, dict(pl)) + self.assertIsInstance(pl2, dict) + self.assertIsInstance(pl2['aDict'], dict) + def test_controlcharacters(self): for i in range(128): c = chr(i) diff --git a/Lib/test/test_profiling/test_sampling_profiler/mocks.py b/Lib/test/test_profiling/test_sampling_profiler/mocks.py index 4e0f7a87c6da54..6ac2d08e898d81 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/mocks.py +++ b/Lib/test/test_profiling/test_sampling_profiler/mocks.py @@ -91,3 +91,22 @@ def __init__(self, thread_id, awaited_by): def __repr__(self): return f"MockAwaitedInfo(thread_id={self.thread_id}, awaited_by={len(self.awaited_by)} tasks)" + + +def make_diff_collector_with_mock_baseline(baseline_samples): + """Create a DiffFlamegraphCollector with baseline injected directly, + skipping the binary round-trip that _load_baseline normally does.""" + from profiling.sampling.stack_collector import ( + DiffFlamegraphCollector, + FlamegraphCollector, + ) + + baseline = FlamegraphCollector(1000) + for sample in baseline_samples: + baseline.collect(sample) + + # Path is unused since we inject _baseline_collector directly; + # use __file__ as a dummy path that passes the existence check. + diff = DiffFlamegraphCollector(1000, baseline_binary_path=__file__) + diff._baseline_collector = baseline + return diff diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py index 033a533fe5444e..29f83c843561cd 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py @@ -18,9 +18,11 @@ THREAD_STATUS_UNKNOWN, THREAD_STATUS_GIL_REQUESTED, THREAD_STATUS_HAS_EXCEPTION, + THREAD_STATUS_MAIN_THREAD, ) from profiling.sampling.binary_collector import BinaryCollector from profiling.sampling.binary_reader import BinaryReader + from profiling.sampling.gecko_collector import GeckoCollector ZSTD_AVAILABLE = _remote_debugging.zstd_available() except ImportError: @@ -318,6 +320,7 @@ def test_status_flags_preserved(self): THREAD_STATUS_UNKNOWN, THREAD_STATUS_GIL_REQUESTED, THREAD_STATUS_HAS_EXCEPTION, + THREAD_STATUS_MAIN_THREAD, THREAD_STATUS_HAS_GIL | THREAD_STATUS_ON_CPU, THREAD_STATUS_HAS_GIL | THREAD_STATUS_HAS_EXCEPTION, THREAD_STATUS_HAS_GIL @@ -342,6 +345,35 @@ def test_status_flags_preserved(self): self.assertEqual(count, len(statuses)) self.assert_samples_equal(samples, collector) + def test_binary_replay_preserves_main_thread_for_gecko(self): + """Binary replay preserves main thread identity for GeckoCollector.""" + samples = [ + [ + make_interpreter( + 0, + [ + make_thread( + 1, + [make_frame("main.py", 10, "main")], + THREAD_STATUS_MAIN_THREAD, + ), + make_thread(2, [make_frame("worker.py", 20, "worker")]), + ], + ) + ] + ] + filename = self.create_binary_file(samples) + collector = GeckoCollector(1000) + + with BinaryReader(filename) as reader: + count = reader.replay_samples(collector) + + self.assertEqual(count, 2) + profile = collector._build_profile() + threads = {thread["tid"]: thread for thread in profile["threads"]} + self.assertTrue(threads[1]["isMainThread"]) + self.assertFalse(threads[2]["isMainThread"]) + def test_multiple_threads_per_sample(self): """Multiple threads in one sample roundtrip exactly.""" threads = [ diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py index 8e6afa91e89daf..503430ddf02163 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -28,6 +28,7 @@ THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU, THREAD_STATUS_GIL_REQUESTED, + THREAD_STATUS_MAIN_THREAD, ) except ImportError: raise unittest.SkipTest( @@ -36,10 +37,26 @@ from test.support import captured_stdout, captured_stderr -from .mocks import MockFrameInfo, MockThreadInfo, MockInterpreterInfo, LocationInfo +from .mocks import MockFrameInfo, MockThreadInfo, MockInterpreterInfo, LocationInfo, make_diff_collector_with_mock_baseline from .helpers import close_and_unlink +def resolve_name(node, strings): + """Resolve a flamegraph node's name from the string table.""" + idx = node.get("name", 0) + if isinstance(idx, int) and 0 <= idx < len(strings): + return strings[idx] + return str(idx) + + +def find_child_by_name(children, strings, substr): + """Find a child node whose resolved name contains substr.""" + for child in children: + if substr in resolve_name(child, strings): + return child + return None + + class TestSampleProfilerComponents(unittest.TestCase): """Unit tests for individual profiler components.""" @@ -397,13 +414,7 @@ def test_flamegraph_collector_basic(self): data = collector._convert_to_flamegraph_format() # With string table, name is now an index - resolve it using the strings array strings = data.get("strings", []) - name_index = data.get("name", 0) - resolved_name = ( - strings[name_index] - if isinstance(name_index, int) and 0 <= name_index < len(strings) - else str(name_index) - ) - self.assertIn(resolved_name, ("No Data", "No significant data")) + self.assertIn(resolve_name(data, strings), ("No Data", "No significant data")) # Test collecting sample data test_frames = [ @@ -422,27 +433,16 @@ def test_flamegraph_collector_basic(self): data = collector._convert_to_flamegraph_format() # Expect promotion: root is the single child (func2), with func1 as its only child strings = data.get("strings", []) - name_index = data.get("name", 0) - name = ( - strings[name_index] - if isinstance(name_index, int) and 0 <= name_index < len(strings) - else str(name_index) - ) - self.assertIsInstance(name, str) + name = resolve_name(data, strings) self.assertTrue(name.startswith("Program Root: ")) - self.assertIn("func2 (file.py:20)", name) # formatted name + self.assertIn("func2 (file.py:20)", name) + self.assertEqual(data["self"], 0) # non-leaf: no self time children = data.get("children", []) self.assertEqual(len(children), 1) child = children[0] - child_name_index = child.get("name", 0) - child_name = ( - strings[child_name_index] - if isinstance(child_name_index, int) - and 0 <= child_name_index < len(strings) - else str(child_name_index) - ) - self.assertIn("func1 (file.py:10)", child_name) # formatted name + self.assertIn("func1 (file.py:10)", resolve_name(child, strings)) self.assertEqual(child["value"], 1) + self.assertEqual(child["self"], 1) # leaf: all time is self def test_flamegraph_collector_export(self): """Test flamegraph HTML export functionality.""" @@ -524,6 +524,7 @@ def test_gecko_collector_basic(self): MockThreadInfo( 1, [MockFrameInfo("file.py", 10, "func1"), MockFrameInfo("file.py", 20, "func2")], + status=THREAD_STATUS_MAIN_THREAD, ) ], ) @@ -556,6 +557,7 @@ def test_gecko_collector_basic(self): threads = profile_data["threads"] self.assertEqual(len(threads), 1) thread_data = threads[0] + self.assertTrue(thread_data["isMainThread"]) # Verify thread structure self.assertIn("samples", thread_data) @@ -1208,6 +1210,463 @@ def test_flamegraph_collector_per_thread_gc_percentage(self): self.assertEqual(collector.per_thread_stats[2]["total"], 6) self.assertAlmostEqual(per_thread_stats[2]["gc_pct"], 10.0, places=1) + def test_diff_flamegraph_identical_profiles(self): + """When baseline and current are identical, diff should be ~0.""" + test_frames = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ]) + ]) + ] + + diff = make_diff_collector_with_mock_baseline([test_frames] * 3) + for _ in range(3): + diff.collect(test_frames) + + data = diff._convert_to_flamegraph_format() + strings = data.get("strings", []) + + self.assertTrue(data["stats"]["is_differential"]) + self.assertEqual(data["stats"]["baseline_samples"], 3) + self.assertEqual(data["stats"]["current_samples"], 3) + self.assertAlmostEqual(data["stats"]["baseline_scale"], 1.0) + + children = data.get("children", []) + self.assertEqual(len(children), 1) + child = children[0] + self.assertIn("func1", resolve_name(child, strings)) + self.assertEqual(child["self_time"], 3) + self.assertAlmostEqual(child["baseline"], 3.0) + self.assertAlmostEqual(child["diff"], 0.0, places=1) + self.assertAlmostEqual(child["diff_pct"], 0.0, places=1) + + self.assertEqual(data["stats"]["elided_count"], 0) + self.assertNotIn("elided_flamegraph", data["stats"]) + + def test_diff_flamegraph_new_function(self): + """A function only in current should have diff_pct=100 and baseline=0.""" + baseline_frames = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ]) + ]) + ] + + diff = make_diff_collector_with_mock_baseline([baseline_frames]) + diff.collect([ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 30, "new_func"), + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ]) + ]) + ]) + + data = diff._convert_to_flamegraph_format() + strings = data.get("strings", []) + + children = data.get("children", []) + self.assertEqual(len(children), 1) + func1_node = children[0] + self.assertIn("func1", resolve_name(func1_node, strings)) + + func1_children = func1_node.get("children", []) + self.assertEqual(len(func1_children), 1) + new_func_node = func1_children[0] + self.assertIn("new_func", resolve_name(new_func_node, strings)) + self.assertEqual(new_func_node["baseline"], 0) + self.assertGreater(new_func_node["self_time"], 0) + self.assertEqual(new_func_node["diff"], new_func_node["self_time"]) + self.assertAlmostEqual(new_func_node["diff_pct"], 100.0) + + def test_diff_flamegraph_changed_functions(self): + """Functions with different sample counts should have correct diff and diff_pct.""" + hot_leaf_sample = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 10, "hot_leaf"), + MockFrameInfo("file.py", 20, "caller"), + ]) + ]) + ] + cold_leaf_sample = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 30, "cold_leaf"), + MockFrameInfo("file.py", 20, "caller"), + ]) + ]) + ] + + # Baseline: 2 samples, current: 4, scale = 2.0 + diff = make_diff_collector_with_mock_baseline( + [hot_leaf_sample, cold_leaf_sample] + ) + for _ in range(3): + diff.collect(hot_leaf_sample) + diff.collect(cold_leaf_sample) + + data = diff._convert_to_flamegraph_format() + strings = data.get("strings", []) + self.assertAlmostEqual(data["stats"]["baseline_scale"], 2.0) + + children = data.get("children", []) + hot_node = find_child_by_name(children, strings, "hot_leaf") + cold_node = find_child_by_name(children, strings, "cold_leaf") + self.assertIsNotNone(hot_node) + self.assertIsNotNone(cold_node) + + # hot_leaf regressed (+50%) + self.assertAlmostEqual(hot_node["baseline"], 2.0) + self.assertEqual(hot_node["self_time"], 3) + self.assertAlmostEqual(hot_node["diff"], 1.0) + self.assertAlmostEqual(hot_node["diff_pct"], 50.0) + + # cold_leaf improved (-50%) + self.assertAlmostEqual(cold_node["baseline"], 2.0) + self.assertEqual(cold_node["self_time"], 1) + self.assertAlmostEqual(cold_node["diff"], -1.0) + self.assertAlmostEqual(cold_node["diff_pct"], -50.0) + + def test_diff_flamegraph_scale_factor(self): + """Scale factor adjusts when sample counts differ.""" + baseline_frames = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ]) + ]) + ] + + diff = make_diff_collector_with_mock_baseline([baseline_frames]) + for _ in range(4): + diff.collect(baseline_frames) + + data = diff._convert_to_flamegraph_format() + self.assertAlmostEqual(data["stats"]["baseline_scale"], 4.0) + + children = data.get("children", []) + self.assertEqual(len(children), 1) + func1_node = children[0] + self.assertEqual(func1_node["self_time"], 4) + self.assertAlmostEqual(func1_node["baseline"], 4.0) + self.assertAlmostEqual(func1_node["diff"], 0.0) + self.assertAlmostEqual(func1_node["diff_pct"], 0.0) + + def test_diff_flamegraph_elided_stacks(self): + """Paths in baseline but not current produce elided stacks.""" + baseline_frames_1 = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ]) + ]) + ] + baseline_frames_2 = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 30, "old_func"), + MockFrameInfo("file.py", 20, "func2"), + ]) + ]) + ] + + diff = make_diff_collector_with_mock_baseline([baseline_frames_1, baseline_frames_2]) + for _ in range(2): + diff.collect(baseline_frames_1) + + data = diff._convert_to_flamegraph_format() + + self.assertGreater(data["stats"]["elided_count"], 0) + self.assertIn("elided_flamegraph", data["stats"]) + elided = data["stats"]["elided_flamegraph"] + self.assertTrue(elided["stats"]["is_differential"]) + self.assertIn("strings", elided) + + elided_strings = elided.get("strings", []) + children = elided.get("children", []) + self.assertEqual(len(children), 1) + child = children[0] + self.assertIn("old_func", resolve_name(child, elided_strings)) + self.assertEqual(child["self_time"], 0) + self.assertAlmostEqual(child["diff_pct"], -100.0) + self.assertGreater(child["baseline"], 0) + self.assertAlmostEqual(child["diff"], -child["baseline"]) + + def test_diff_flamegraph_function_matched_despite_line_change(self): + """Functions match by (filename, funcname), ignoring lineno.""" + baseline_frames = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ]) + ]) + ] + + diff = make_diff_collector_with_mock_baseline([baseline_frames]) + # Same functions but different line numbers + diff.collect([ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 99, "func1"), + MockFrameInfo("file.py", 55, "func2"), + ]) + ]) + ]) + + data = diff._convert_to_flamegraph_format() + strings = data.get("strings", []) + + children = data.get("children", []) + self.assertEqual(len(children), 1) + child = children[0] + self.assertIn("func1", resolve_name(child, strings)) + self.assertGreater(child["baseline"], 0) + self.assertGreater(child["self_time"], 0) + self.assertAlmostEqual(child["diff"], 0.0, places=1) + self.assertAlmostEqual(child["diff_pct"], 0.0, places=1) + + def test_diff_flamegraph_empty_current(self): + """Empty current profile still produces differential metadata and elided paths.""" + baseline_frames = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [MockFrameInfo("file.py", 10, "func1")]) + ]) + ] + + diff = make_diff_collector_with_mock_baseline([baseline_frames]) + # Don't collect anything in current + + data = diff._convert_to_flamegraph_format() + self.assertIn("name", data) + self.assertEqual(data["value"], 0) + # Differential metadata should still be populated + self.assertTrue(data["stats"]["is_differential"]) + # All baseline paths should be elided since current is empty + self.assertGreater(data["stats"]["elided_count"], 0) + + def test_diff_flamegraph_empty_baseline(self): + """Empty baseline with non-empty current uses scale=1.0 fallback.""" + diff = make_diff_collector_with_mock_baseline([]) + diff.collect([ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ]) + ]) + ]) + + data = diff._convert_to_flamegraph_format() + strings = data.get("strings", []) + + self.assertTrue(data["stats"]["is_differential"]) + self.assertEqual(data["stats"]["baseline_samples"], 0) + self.assertEqual(data["stats"]["current_samples"], 1) + self.assertAlmostEqual(data["stats"]["baseline_scale"], 1.0) + self.assertEqual(data["stats"]["elided_count"], 0) + + children = data.get("children", []) + self.assertEqual(len(children), 1) + child = children[0] + self.assertIn("func1", resolve_name(child, strings)) + self.assertEqual(child["self_time"], 1) + self.assertAlmostEqual(child["baseline"], 0.0) + self.assertAlmostEqual(child["diff"], 1.0) + self.assertAlmostEqual(child["diff_pct"], 100.0) + + def test_diff_flamegraph_export(self): + """DiffFlamegraphCollector export produces differential HTML.""" + test_frames = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ]) + ]) + ] + + diff = make_diff_collector_with_mock_baseline([test_frames]) + diff.collect(test_frames) + + flamegraph_out = tempfile.NamedTemporaryFile( + suffix=".html", delete=False + ) + self.addCleanup(close_and_unlink, flamegraph_out) + + with captured_stdout(), captured_stderr(): + diff.export(flamegraph_out.name) + + self.assertTrue(os.path.exists(flamegraph_out.name)) + self.assertGreater(os.path.getsize(flamegraph_out.name), 0) + + with open(flamegraph_out.name, "r", encoding="utf-8") as f: + content = f.read() + + self.assertIn("", content.lower()) + self.assertIn("Differential Flamegraph", content) + self.assertIn('"is_differential": true', content) + self.assertIn("d3-flame-graph", content) + self.assertIn('id="diff-legend-section"', content) + self.assertIn("Differential Colors", content) + + def test_diff_flamegraph_preserves_metadata(self): + """Differential mode preserves threads and opcodes metadata.""" + test_frames = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [MockFrameInfo("a.py", 10, "func_a", opcode=100)]), + MockThreadInfo(2, [MockFrameInfo("b.py", 20, "func_b", opcode=200)]), + ]) + ] + + diff = make_diff_collector_with_mock_baseline([test_frames]) + diff.collect(test_frames) + + data = diff._convert_to_flamegraph_format() + strings = data.get("strings", []) + + self.assertTrue(data["stats"]["is_differential"]) + + self.assertIn("threads", data) + self.assertEqual(len(data["threads"]), 2) + + children = data.get("children", []) + self.assertEqual(len(children), 2) + + opcodes_found = set() + for child in children: + self.assertIn("diff", child) + self.assertIn("diff_pct", child) + self.assertIn("baseline", child) + self.assertIn("self_time", child) + self.assertIn("threads", child) + + if "opcodes" in child: + opcodes_found.update(child["opcodes"].keys()) + + self.assertIn(100, opcodes_found) + self.assertIn(200, opcodes_found) + + self.assertIn("per_thread_stats", data["stats"]) + per_thread_stats = data["stats"]["per_thread_stats"] + self.assertIn(1, per_thread_stats) + self.assertIn(2, per_thread_stats) + + def test_diff_flamegraph_elided_preserves_metadata(self): + """Elided flamegraph preserves thread_stats, per_thread_stats, and opcodes.""" + baseline_frames_1 = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 10, "func1", opcode=100), + MockFrameInfo("file.py", 20, "func2", opcode=101), + ], status=THREAD_STATUS_HAS_GIL) + ]) + ] + baseline_frames_2 = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 30, "old_func", opcode=200), + MockFrameInfo("file.py", 20, "func2", opcode=101), + ], status=THREAD_STATUS_HAS_GIL) + ]) + ] + + diff = make_diff_collector_with_mock_baseline([baseline_frames_1, baseline_frames_2]) + for _ in range(2): + diff.collect(baseline_frames_1) + + data = diff._convert_to_flamegraph_format() + elided = data["stats"]["elided_flamegraph"] + + self.assertTrue(elided["stats"]["is_differential"]) + self.assertIn("thread_stats", elided["stats"]) + self.assertIn("per_thread_stats", elided["stats"]) + self.assertIn("baseline_samples", elided["stats"]) + self.assertIn("current_samples", elided["stats"]) + self.assertIn("strings", elided) + + elided_strings = elided.get("strings", []) + children = elided.get("children", []) + self.assertEqual(len(children), 1) + old_func_node = children[0] + if "opcodes" in old_func_node: + self.assertIn(200, old_func_node["opcodes"]) + self.assertEqual(old_func_node["self_time"], 0) + self.assertAlmostEqual(old_func_node["diff_pct"], -100.0) + + def test_diff_flamegraph_load_baseline(self): + """Diff annotations work when baseline is loaded from a binary file.""" + from profiling.sampling.binary_collector import BinaryCollector + from profiling.sampling.stack_collector import DiffFlamegraphCollector + from .test_binary_format import make_frame, make_thread, make_interpreter + + hot_sample = [make_interpreter(0, [make_thread(1, [ + make_frame("file.py", 10, "hot_leaf"), + make_frame("file.py", 20, "caller"), + ])])] + cold_sample = [make_interpreter(0, [make_thread(1, [ + make_frame("file.py", 30, "cold_leaf"), + make_frame("file.py", 20, "caller"), + ])])] + + # Baseline: 2 samples, current: 4, scale = 2.0 + bin_file = tempfile.NamedTemporaryFile(suffix=".bin", delete=False) + self.addCleanup(close_and_unlink, bin_file) + + writer = BinaryCollector( + bin_file.name, sample_interval_usec=1000, compression='none' + ) + writer.collect(hot_sample) + writer.collect(cold_sample) + writer.export(None) + + diff = DiffFlamegraphCollector( + 1000, baseline_binary_path=bin_file.name + ) + hot_mock = [MockInterpreterInfo(0, [MockThreadInfo(1, [ + MockFrameInfo("file.py", 10, "hot_leaf"), + MockFrameInfo("file.py", 20, "caller"), + ])])] + cold_mock = [MockInterpreterInfo(0, [MockThreadInfo(1, [ + MockFrameInfo("file.py", 30, "cold_leaf"), + MockFrameInfo("file.py", 20, "caller"), + ])])] + for _ in range(3): + diff.collect(hot_mock) + diff.collect(cold_mock) + + data = diff._convert_to_flamegraph_format() + strings = data.get("strings", []) + + self.assertTrue(data["stats"]["is_differential"]) + self.assertAlmostEqual(data["stats"]["baseline_scale"], 2.0) + + children = data.get("children", []) + hot_node = find_child_by_name(children, strings, "hot_leaf") + cold_node = find_child_by_name(children, strings, "cold_leaf") + self.assertIsNotNone(hot_node) + self.assertIsNotNone(cold_node) + + # hot_leaf regressed (+50%) + self.assertAlmostEqual(hot_node["baseline"], 2.0) + self.assertEqual(hot_node["self_time"], 3) + self.assertAlmostEqual(hot_node["diff"], 1.0) + self.assertAlmostEqual(hot_node["diff_pct"], 50.0) + + # cold_leaf improved (-50%) + self.assertAlmostEqual(cold_node["baseline"], 2.0) + self.assertEqual(cold_node["self_time"], 1) + self.assertAlmostEqual(cold_node["diff"], -1.0) + self.assertAlmostEqual(cold_node["diff_pct"], -50.0) + class TestRecursiveFunctionHandling(unittest.TestCase): """Tests for correct handling of recursive functions in cumulative stats.""" diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index c67bfc67479985..cace780f79f515 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -854,6 +854,8 @@ class ExternalEntityParserCreateErrorTest(unittest.TestCase): def setUpClass(cls): cls.testcapi = import_helper.import_module('_testcapi') + @unittest.skipIf(support.Py_TRACE_REFS, + 'Py_TRACE_REFS conflicts with testcapi.set_nomemory') def test_error_path_no_crash(self): # When an allocation inside ExternalEntityParserCreate fails, # the partially-initialized subparser is deallocated. This diff --git a/Lib/test/test_pyrepl/test_fancycompleter.py b/Lib/test/test_pyrepl/test_fancycompleter.py new file mode 100644 index 00000000000000..77c80853a3c0e3 --- /dev/null +++ b/Lib/test/test_pyrepl/test_fancycompleter.py @@ -0,0 +1,247 @@ +import importlib +import os +import types +import unittest + +from _colorize import ANSIColors, get_theme +from _pyrepl.completing_reader import stripcolor +from _pyrepl.fancycompleter import Completer, commonprefix +from test.support.import_helper import ready_to_import + +class MockPatch: + def __init__(self): + self.original_values = {} + + def setattr(self, obj, name, value): + if obj not in self.original_values: + self.original_values[obj] = {} + if name not in self.original_values[obj]: + self.original_values[obj][name] = getattr(obj, name) + setattr(obj, name, value) + + def restore_all(self): + for obj, attrs in self.original_values.items(): + for name, value in attrs.items(): + setattr(obj, name, value) + +class FancyCompleterTests(unittest.TestCase): + def setUp(self): + self.mock_patch = MockPatch() + + def tearDown(self): + self.mock_patch.restore_all() + + def test_commonprefix(self): + self.assertEqual(commonprefix(['isalpha', 'isdigit', 'foo']), '') + self.assertEqual(commonprefix(['isalpha', 'isdigit']), 'is') + self.assertEqual(commonprefix([]), '') + + def test_complete_attribute(self): + compl = Completer({'a': None}, use_colors=False) + self.assertEqual(compl.attr_matches('a.'), ['a.__']) + matches = compl.attr_matches('a.__') + self.assertNotIn('__class__', matches) + self.assertIn('a.__class__', matches) + match = compl.attr_matches('a.__class') + self.assertEqual(len(match), 1) + self.assertTrue(match[0].startswith('a.__class__')) + + def test_complete_attribute_prefix(self): + class C(object): + attr = 1 + _attr = 2 + __attr__attr = 3 + compl = Completer({'a': C}, use_colors=False) + self.assertEqual(compl.attr_matches('a.'), ['a.attr', 'a.mro']) + self.assertEqual( + compl.attr_matches('a._'), + ['a._C__attr__attr', 'a._attr', ' '], + ) + matches = compl.attr_matches('a.__') + self.assertNotIn('__class__', matches) + self.assertIn('a.__class__', matches) + match = compl.attr_matches('a.__class') + self.assertEqual(len(match), 1) + self.assertTrue(match[0].startswith('a.__class__')) + + compl = Completer({'a': None}, use_colors=False) + self.assertEqual(compl.attr_matches('a._'), ['a.__']) + + def test_complete_attribute_colored(self): + theme = get_theme() + compl = Completer({'a': 42}, use_colors=True) + matches = compl.attr_matches('a.__') + self.assertGreater(len(matches), 2) + expected_color = theme.fancycompleter.type + expected_part = f'{expected_color}a.__class__{ANSIColors.RESET}' + for match in matches: + if expected_part in match: + break + else: + self.assertFalse(True, matches) + self.assertIn(' ', matches) + + def test_preserves_callable_postfix_for_single_attribute_match(self): + compl = Completer({'os': os}, use_colors=False) + self.assertEqual(compl.attr_matches('os.getpid'), ['os.getpid()']) + + def test_property_method_not_called(self): + class Foo: + property_called = False + + @property + def bar(self): + self.property_called = True + return 1 + + foo = Foo() + compl = Completer({'foo': foo}, use_colors=False) + self.assertEqual(compl.attr_matches('foo.b'), ['foo.bar']) + self.assertFalse(foo.property_called) + + def test_excessive_getattr(self): + class Foo: + calls = 0 + bar = '' + + def __getattribute__(self, name): + if name == 'bar': + self.calls += 1 + return None + return super().__getattribute__(name) + + foo = Foo() + compl = Completer({'foo': foo}, use_colors=False) + self.assertEqual(compl.complete('foo.b', 0), 'foo.bar') + self.assertEqual(foo.calls, 1) + + def test_uncreated_attr(self): + class Foo: + __slots__ = ('bar',) + + compl = Completer({'foo': Foo()}, use_colors=False) + self.assertEqual(compl.complete('foo.', 0), 'foo.bar') + + def test_module_attributes_do_not_reify_lazy_imports(self): + with ready_to_import("test_pyrepl_lazy_mod", "lazy import json\n") as (name, _): + module = importlib.import_module(name) + self.assertIs(type(module.__dict__["json"]), types.LazyImportType) + + compl = Completer({name: module}, use_colors=False) + self.assertEqual(compl.attr_matches(f"{name}.j"), [f"{name}.json"]) + self.assertIs(type(module.__dict__["json"]), types.LazyImportType) + + def test_complete_colored_single_match(self): + """No coloring, via commonprefix.""" + compl = Completer({'foobar': 42}, use_colors=True) + matches = compl.global_matches('foob') + self.assertEqual(matches, ['foobar']) + + def test_does_not_color_single_match(self): + class obj: + msgs = [] + + compl = Completer({'obj': obj}, use_colors=True) + matches = compl.attr_matches('obj.msgs') + self.assertEqual(matches, ['obj.msgs']) + + def test_complete_global(self): + compl = Completer({'foobar': 1, 'foobazzz': 2}, use_colors=False) + self.assertEqual(compl.global_matches('foo'), ['fooba']) + matches = compl.global_matches('fooba') + self.assertEqual(set(matches), set(['foobar', 'foobazzz'])) + self.assertEqual(compl.global_matches('foobaz'), ['foobazzz']) + self.assertEqual(compl.global_matches('nothing'), []) + + def test_complete_global_colored(self): + theme = get_theme() + compl = Completer({'foobar': 1, 'foobazzz': 2}, use_colors=True) + self.assertEqual(compl.global_matches('foo'), ['fooba']) + matches = compl.global_matches('fooba') + + # these are the fake escape sequences which are needed so that + # readline displays the matches in the proper order + N0 = f"\x1b[000;00m" + N1 = f"\x1b[000;01m" + int_color = theme.fancycompleter.int + self.assertEqual(set(matches), { + ' ', + f'{N0}{int_color}foobar{ANSIColors.RESET}', + f'{N1}{int_color}foobazzz{ANSIColors.RESET}', + }) + self.assertEqual(compl.global_matches('foobaz'), ['foobazzz']) + self.assertEqual(compl.global_matches('nothing'), []) + + def test_large_color_sort_prefix_is_stripped(self): + compl = Completer({'a': 42}, use_colors=True) + match = compl._color_for_obj(1000, 'spam', 1) + self.assertEqual(stripcolor(match), 'spam') + + def test_complete_with_indexer(self): + compl = Completer({'lst': [None, 2, 3]}, use_colors=False) + self.assertEqual(compl.attr_matches('lst[0].'), ['lst[0].__']) + matches = compl.attr_matches('lst[0].__') + self.assertNotIn('__class__', matches) + self.assertIn('lst[0].__class__', matches) + match = compl.attr_matches('lst[0].__class') + self.assertEqual(len(match), 1) + self.assertTrue(match[0].startswith('lst[0].__class__')) + + def test_autocomplete(self): + class A: + aaa = None + abc_1 = None + abc_2 = None + abc_3 = None + bbb = None + compl = Completer({'A': A}, use_colors=False) + # + # In this case, we want to display all attributes which start with + # 'a'. Moreover, we also include a space to prevent readline to + # automatically insert the common prefix (which will the the ANSI escape + # sequence if we use colors). + matches = compl.attr_matches('A.a') + self.assertEqual( + sorted(matches), + [' ', 'A.aaa', 'A.abc_1', 'A.abc_2', 'A.abc_3'], + ) + # + # If there is an actual common prefix, we return just it, so that readline + # will insert it into place + matches = compl.attr_matches('A.ab') + self.assertEqual(matches, ['A.abc_']) + # + # Finally, at the next tab, we display again all the completions available + # for this common prefix. Again, we insert a spurious space to prevent the + # automatic completion of ANSI sequences. + matches = compl.attr_matches('A.abc_') + self.assertEqual( + sorted(matches), + [' ', 'A.abc_1', 'A.abc_2', 'A.abc_3'], + ) + + def test_complete_exception(self): + compl = Completer({}, use_colors=False) + self.assertEqual(compl.attr_matches('xxx.'), []) + + def test_complete_invalid_attr(self): + compl = Completer({'str': str}, use_colors=False) + self.assertEqual(compl.attr_matches('str.xx'), []) + + def test_complete_function_skipped(self): + compl = Completer({'str': str}, use_colors=False) + self.assertEqual(compl.attr_matches('str.split().'), []) + + def test_unicode_in___dir__(self): + class Foo(object): + def __dir__(self): + return ['hello', 'world'] + + compl = Completer({'a': Foo()}, use_colors=False) + matches = compl.attr_matches('a.') + self.assertEqual(matches, ['a.hello', 'a.world']) + self.assertIs(type(matches[0]), str) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 082215da0a3fba..c3556823c72476 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -1,3 +1,4 @@ +import contextlib import importlib import io import itertools @@ -13,7 +14,14 @@ from pkgutil import ModuleInfo from unittest import TestCase, skipUnless, skipIf, SkipTest from unittest.mock import Mock, patch -from test.support import force_not_colorized, make_clean_env, Py_DEBUG +import warnings +from test.support import ( + captured_stdout, + captured_stderr, + force_not_colorized, + make_clean_env, + Py_DEBUG, +) from test.support import has_subprocess_support, SHORT_TIMEOUT, STDLIB_DIR from test.support.import_helper import import_module from test.support.os_helper import EnvironmentVarGuard, unlink @@ -33,6 +41,8 @@ ModuleCompleter, HARDCODED_SUBMODULES, ) +from _pyrepl.fancycompleter import Completer as FancyCompleter +import _pyrepl.readline as pyrepl_readline from _pyrepl.readline import ( ReadlineAlikeReader, ReadlineConfig, @@ -44,6 +54,14 @@ import pty except ImportError: pty = None +try: + import readline as readline_module +except ImportError: + readline_module = None +try: + import tkinter +except ImportError: + tkinter = None class ReplTestCase(TestCase): @@ -937,6 +955,92 @@ def test_func(self): self.assertEqual(mock_stderr.getvalue(), "") +class TestPyReplFancyCompleter(TestCase): + def prepare_reader(self, events, namespace, *, use_colors): + console = FakeConsole(events) + config = ReadlineConfig() + config.readline_completer = FancyCompleter( + namespace, use_colors=use_colors + ).complete + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_simple_completion_preserves_callable_postfix(self): + events = code_to_events("os.getpid\t\n") + + namespace = {"os": os} + reader = self.prepare_reader(events, namespace, use_colors=False) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.getpid()") + + def test_attribute_menu_tracks_typed_stem(self): + class Obj: + apple = 1 + apricot = 2 + banana = 3 + + namespace = {"obj": Obj} + reader = self.prepare_reader( + code_to_events("obj.\t\ta"), + namespace, + use_colors=True, + ) + + with self.assertRaises(StopIteration): + while True: + reader.handle1() + + self.assertEqual("".join(reader.buffer), "obj.a") + self.assertTrue(reader.cmpltn_menu_visible) + menu = "\n".join(reader.cmpltn_menu) + self.assertIn("apple", menu) + self.assertIn("apricot", menu) + self.assertNotIn("banana", menu) + self.assertNotIn("mro", menu) + + +class TestPyReplReadlineSetup(TestCase): + def test_setup_ignores_basic_completer_env_when_env_is_disabled(self): + class FakeFancyCompleter: + def __init__(self, namespace): + self.namespace = namespace + + def complete(self, text, state): + return None + + class FakeBasicCompleter(FakeFancyCompleter): + pass + + wrapper = Mock() + wrapper.config = ReadlineConfig() + stdin = Mock() + stdout = Mock() + stdin.fileno.return_value = 0 + stdout.fileno.return_value = 1 + + with ( + patch.object(pyrepl_readline, "_wrapper", wrapper), + patch.object(pyrepl_readline, "raw_input", None), + patch.object(pyrepl_readline, "FancyCompleter", FakeFancyCompleter), + patch.object(pyrepl_readline, "RLCompleter", FakeBasicCompleter), + patch.object(pyrepl_readline.sys, "stdin", stdin), + patch.object(pyrepl_readline.sys, "stdout", stdout), + patch.object(pyrepl_readline.sys, "flags", Mock(ignore_environment=True)), + patch.object(pyrepl_readline.os, "isatty", return_value=True), + patch.object(pyrepl_readline.os, "getenv") as mock_getenv, + patch("builtins.input", lambda prompt="": prompt), + ): + mock_getenv.return_value = "1" + pyrepl_readline._setup({}) + + self.assertIsInstance( + wrapper.config.readline_completer.__self__, + FakeFancyCompleter, + ) + mock_getenv.assert_not_called() + + class TestPyReplModuleCompleter(TestCase): def setUp(self): # Make iter_modules() search only the standard library. @@ -958,7 +1062,9 @@ def prepare_reader(self, events, namespace): reader = ReadlineAlikeReader(console=console, config=config) return reader - def test_import_completions(self): + @patch.dict(sys.modules, + {"importlib.resources": object()}) # don't propose to import it + def test_completions(self): cases = ( ("import path\t\n", "import pathlib"), ("import importlib.\t\tres\t\n", "import importlib.resources"), @@ -1012,7 +1118,7 @@ def test_sub_module_private_completions(self): # Return public methods by default ("from foo import \t\n", "from foo import public"), # Return private methods if explicitly specified - ("from foo import _\t\n", "from foo import _private"), + ("from foo import _p\t\n", "from foo import _private"), ) for code, expected in cases: with self.subTest(code=code): @@ -1033,12 +1139,13 @@ def test_builtin_completion_top_level(self): output = reader.readline() self.assertEqual(output, expected) - def test_relative_import_completions(self): + def test_relative_completions(self): cases = ( (None, "from .readl\t\n", "from .readl"), (None, "from . import readl\t\n", "from . import readl"), ("_pyrepl", "from .readl\t\n", "from .readline"), ("_pyrepl", "from . import readl\t\n", "from . import readline"), + ("_pyrepl", "from .readline import mul\t\n", "from .readline import multiline_input"), ("_pyrepl", "from .. import toodeep\t\n", "from .. import toodeep"), ("concurrent", "from .futures.i\t\n", "from .futures.interpreter"), ) @@ -1070,7 +1177,7 @@ def test_no_fallback_on_regular_completion(self): cases = ( ("import pri\t\n", "import pri"), ("from pri\t\n", "from pri"), - ("from typing import Na\t\n", "from typing import Na"), + ("from typong import Na\t\n", "from typong import Na"), ) for code, expected in cases: with self.subTest(code=code): @@ -1083,8 +1190,8 @@ def test_global_cache(self): with (tempfile.TemporaryDirectory() as _dir1, patch.object(sys, "path", [_dir1, *sys.path])): dir1 = pathlib.Path(_dir1) - (dir1 / "mod_aa.py").mkdir() - (dir1 / "mod_bb.py").mkdir() + (dir1 / "mod_aa.py").touch() + (dir1 / "mod_bb.py").touch() events = code_to_events("import mod_a\t\nimport mod_b\t\n") reader = self.prepare_reader(events, namespace={}) output_1, output_2 = reader.readline(), reader.readline() @@ -1094,7 +1201,8 @@ def test_global_cache(self): def test_hardcoded_stdlib_submodules(self): cases = ( ("import collections.\t\n", "import collections.abc"), - ("from os import \t\n", "from os import path"), + ("import os.\t\n", "import os.path"), + ("import math.\t\n", "import math.integer"), ("import xml.parsers.expat.\t\te\t\n\n", "import xml.parsers.expat.errors"), ("from xml.parsers.expat import \t\tm\t\n\n", "from xml.parsers.expat import model"), ) @@ -1207,6 +1315,115 @@ def test_already_imported_module_without_origin_or_spec(self): self.assertEqual(output, f"import {mod}.") del sys.modules[mod] + @patch.dict(sys.modules) + def test_attribute_completion(self): + with tempfile.TemporaryDirectory() as _dir: + dir = pathlib.Path(_dir) + (dir / "foo.py").write_text("bar = 42") + (dir / "bar.py").write_text("baz = 42") + (dir / "pack").mkdir() + (dir / "pack" / "__init__.py").write_text("attr = 42") + (dir / "pack" / "foo.py").touch() + (dir / "pack" / "bar.py").touch() + (dir / "pack" / "baz.py").touch() + sys.modules.pop("graphlib", None) # test modules may have been imported by previous tests + sys.modules.pop("antigravity", None) + sys.modules.pop("unittest.__main__", None) + with patch.object(sys, "path", [_dir, *sys.path]): + pkgutil.get_importer(_dir).invalidate_caches() + importlib.import_module("bar") + cases = ( + # needs 2 tabs to import (show prompt, then import) + ("from foo import \t\n", "from foo import ", set()), + ("from foo import \t\t\n", "from foo import bar", {"foo"}), + ("from foo import ba\t\n", "from foo import ba", set()), + ("from foo import ba\t\t\n", "from foo import bar", {"foo"}), + # reset if a character is inserted between tabs + ("from foo import \tb\ta\t\n", "from foo import ba", set()), + # packages: needs 3 tabs ([ not unique ], prompt, import) + ("from pack import \t\t\n", "from pack import ", set()), + ("from pack import \t\t\t\n", "from pack import ", {"pack"}), + ("from pack import \t\t\ta\t\n", "from pack import attr", {"pack"}), + # one match: needs 2 tabs (insert + show prompt, import) + ("from pack import f\t\n", "from pack import foo", set()), + ("from pack import f\t\t\n", "from pack import foo", {"pack"}), + # common prefix: needs 3 tabs (insert + [ not unique ], prompt, import) + ("from pack import b\t\n", "from pack import ba", set()), + ("from pack import b\t\t\n", "from pack import ba", set()), + ("from pack import b\t\t\t\n", "from pack import ba", {"pack"}), + # module already imported + ("from bar import b\t\n", "from bar import baz", set()), + # stdlib modules are automatically imported + ("from graphlib import T\t\n", "from graphlib import TopologicalSorter", {"graphlib"}), + # except those with known side-effects + ("from antigravity import g\t\n", "from antigravity import g", set()), + ("from unittest.__main__ import \t\n", "from unittest.__main__ import ", set()), + ) + for code, expected, expected_imports in cases: + with self.subTest(code=code), patch.dict(sys.modules): + _imported = set(sys.modules.keys()) + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + new_imports = sys.modules.keys() - _imported + self.assertEqual(new_imports, expected_imports) + + @patch.dict(sys.modules) + def test_attribute_completion_error_on_import(self): + with tempfile.TemporaryDirectory() as _dir: + dir = pathlib.Path(_dir) + (dir / "foo.py").write_text("bar = 42") + (dir / "boom.py").write_text("1 <> 2") + with patch.object(sys, "path", [_dir, *sys.path]): + cases = ( + ("from boom import \t\t\n", "from boom import "), + ("from foo import \t\t\n", "from foo import bar"), # still working + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + self.assertNotIn("boom", sys.modules) + + @patch.dict(sys.modules) + def test_attribute_completion_error_on_attributes_access(self): + with tempfile.TemporaryDirectory() as _dir: + dir = pathlib.Path(_dir) + (dir / "boom").mkdir() + (dir / "boom"/"__init__.py").write_text("def __dir__(): raise ValueError()") + (dir / "boom"/"submodule.py").touch() + with patch.object(sys, "path", [_dir, *sys.path]): + events = code_to_events("from boom import \t\t\n") # trigger import + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertIn("boom", sys.modules) + # ignore attributes, just propose submodule + self.assertEqual(output, "from boom import submodule") + + @patch.dict(sys.modules) + def test_attribute_completion_private_and_invalid_names(self): + with tempfile.TemporaryDirectory() as _dir: + dir = pathlib.Path(_dir) + (dir / "foo.py").write_text("_secret = 'bar'") + with patch.object(sys, "path", [_dir, *sys.path]): + mod = importlib.import_module("foo") + mod.__dict__["invalid-identifier"] = "baz" + cases = ( + ("from foo import \t\n", "from foo import "), + ("from foo import _s\t\n", "from foo import _secret"), + ("from foo import inv\t\n", "from foo import inv"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + + def test_get_path_and_prefix(self): cases = ( ('', ('', '')), @@ -1338,8 +1555,119 @@ def test_parse_error(self): with self.subTest(code=code): self.assertEqual(actual, None) + @patch.dict(sys.modules) + def test_suggestions_and_messages(self) -> None: + # more unitary tests checking the exact suggestions provided + # (sorting, de-duplication, import action...) + _prompt = ("[ module not imported, press again to import it " + "and propose attributes ]") + _error = "[ error during import: division by zero ]" + with tempfile.TemporaryDirectory() as _dir: + dir = pathlib.Path(_dir) + (dir / "foo.py").write_text("bar = 42") + (dir / "boom.py").write_text("1/0") + (dir / "pack").mkdir() + (dir / "pack" / "__init__.py").write_text("foo = 1; bar = 2;") + (dir / "pack" / "bar.py").touch() + sys.modules.pop("graphlib", None) # test modules may have been imported by previous tests + sys.modules.pop("string.templatelib", None) + with patch.object(sys, "path", [_dir, *sys.path]): + pkgutil.get_importer(_dir).invalidate_caches() + # NOTE: Cases are intentionally sequential and share completer + # state. Earlier cases may import modules that later cases + # depend on. Do NOT reorder without understanding dependencies. + cases = ( + # no match != not an import + ("import nope", ([], None), set()), + ("improt nope", None, set()), + # names sorting + ("import col", (["collections", "colorsys"], None), set()), + # module auto-import + ("import fo", (["foo"], None), set()), + ("from foo import ", ([], (_prompt, None)), {"foo"}), + ("from foo import ", (["bar"], None), set()), # now imported + ("from foo import ba", (["bar"], None), set()), + # error during import + ("from boom import ", ([], (_prompt, _error)), set()), + ("from boom import ", ([], None), set()), # do not retry + # packages + ("from collections import a", (["abc"], None), set()), + ("from pack import ", (["bar"], (_prompt, None)), {"pack"}), + ("from pack import ", (["bar", "foo"], None), set()), + ("from pack.bar import ", ([], (_prompt, None)), {"pack.bar"}), + ("from pack.bar import ", ([], None), set()), + # stdlib = auto-imported + ("from graphlib import T", (["TopologicalSorter"], None), {"graphlib"}), + ("from string.templatelib import c", (["convert"], None), {"string.templatelib"}), + ) + completer = ModuleCompleter() + for i, (code, expected, expected_imports) in enumerate(cases): + with self.subTest(code=code, i=i): + _imported = set(sys.modules.keys()) + result = completer.get_completions(code) + self.assertEqual(result is None, expected is None) + if result: + compl, act = result + self.assertEqual(compl, expected[0]) + self.assertEqual(act is None, expected[1] is None) + if act: + msg, func = act + self.assertEqual(msg, expected[1][0]) + act_result = func() + self.assertEqual(act_result, expected[1][1]) + + new_imports = sys.modules.keys() - _imported + self.assertSetEqual(new_imports, expected_imports) + + +# Audit hook used to check for stdlib modules import side-effects +# Defined globally to avoid adding one hook per test run (refleak) +_audit_events: set[str] | None = None + + +def _hook(name: str, _args: tuple): + if _audit_events is not None: # No-op when not activated + _audit_events.add(name) +sys.addaudithook(_hook) + + +@contextlib.contextmanager +def _capture_audit_events(): + global _audit_events + _audit_events = set() + try: + yield _audit_events + finally: + _audit_events = None + + +class TestModuleCompleterAutomaticImports(TestCase): + def test_no_side_effects(self): + from test.test___all__ import AllTest # TODO: extract to a helper? + + completer = ModuleCompleter() + for _, modname in AllTest().walk_modules(completer._stdlib_path, ""): + with self.subTest(modname=modname): + with (captured_stdout() as out, + captured_stderr() as err, + _capture_audit_events() as audit_events, + (patch("tkinter._tkinter.create") if tkinter + else contextlib.nullcontext()) as tk_mock, + warnings.catch_warnings(action="ignore")): + completer._maybe_import_module(modname) + # Test no module is imported that + # 1. prints any text + self.assertEqual(out.getvalue(), "") + self.assertEqual(err.getvalue(), "") + # 2. spawn any subprocess (eg. webbrowser.open) + self.assertNotIn("subprocess.Popen", audit_events) + # 3. launch a Tk window + if tk_mock is not None: + tk_mock.assert_not_called() + class TestHardcodedSubmodules(TestCase): + @patch.dict(sys.modules) def test_hardcoded_stdlib_submodules_are_importable(self): for parent_path, submodules in HARDCODED_SUBMODULES.items(): for module_name in submodules: @@ -1947,9 +2275,12 @@ def test_no_newline(self): commands = "print('Something pretty long', end='')\nexit()\n" expected_output_sequence = "Something pretty long>>> exit()" - basic_output, basic_exit_code = self.run_repl(commands, env=env) - self.assertEqual(basic_exit_code, 0) - self.assertIn(expected_output_sequence, basic_output) + # gh-143394: The basic REPL needs the readline module to turn off + # ECHO terminal attribute. + if readline_module is not None: + basic_output, basic_exit_code = self.run_repl(commands, env=env) + self.assertEqual(basic_exit_code, 0) + self.assertIn(expected_output_sequence, basic_output) output, exit_code = self.run_repl(commands) self.assertEqual(exit_code, 0) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index fc6694d489fb0f..0946289328ccda 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -571,6 +571,13 @@ def test_single_process(self): self.assertEqual(regrtest.num_workers, 0) self.assertTrue(regrtest.single_process) + def test_pythoninfo(self): + ns = self.parse_args([]) + self.assertFalse(ns.pythoninfo) + + ns = self.parse_args(['--pythoninfo']) + self.assertTrue(ns.pythoninfo) + @dataclasses.dataclass(slots=True) class Rerun: @@ -2427,6 +2434,11 @@ def test_pgo_exclude(self): self.assertNotIn('test_re', tests) self.assertEqual(len(tests), len(pgo_tests) - 1) + def test_pythoninfo(self): + testname = self.create_test() + output = self.run_tests('--pythoninfo', testname) + self.assertIn("Python build information", output) + class TestUtils(unittest.TestCase): def test_format_duration(self): diff --git a/Lib/test/test_sqlite3/test_hooks.py b/Lib/test/test_sqlite3/test_hooks.py index 495ef97fa3c61c..7e5117771b379f 100644 --- a/Lib/test/test_sqlite3/test_hooks.py +++ b/Lib/test/test_sqlite3/test_hooks.py @@ -120,6 +120,21 @@ def test_collation_register_twice(self): self.assertEqual(result[0][0], 'b') self.assertEqual(result[1][0], 'a') + def test_collation_register_when_busy(self): + # See https://github.com/python/cpython/issues/146090. + con = self.con + con.create_collation("mycoll", lambda x, y: (x > y) - (x < y)) + con.execute("CREATE TABLE t(x TEXT)") + con.execute("INSERT INTO t VALUES (?)", ("a",)) + con.execute("INSERT INTO t VALUES (?)", ("b",)) + con.commit() + + cursor = self.con.execute("SELECT x FROM t ORDER BY x COLLATE mycoll") + next(cursor) + # Replace the collation while the statement is active -> SQLITE_BUSY. + with self.assertRaises(sqlite.OperationalError) as cm: + self.con.create_collation("mycoll", lambda a, b: 0) + def test_deregister_collation(self): """ Register a collation, then deregister it. Make sure an error is raised if we try diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index dc795c6bd8a41f..cd90e332443800 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1,5 +1,6 @@ # Test the support for SSL and sockets +import contextlib import sys import unittest import unittest.mock @@ -47,12 +48,13 @@ PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) HOST = socket_helper.HOST +IS_AWS_LC = "AWS-LC" in ssl.OPENSSL_VERSION IS_OPENSSL_3_0_0 = ssl.OPENSSL_VERSION_INFO >= (3, 0, 0) CAN_GET_SELECTED_OPENSSL_GROUP = ssl.OPENSSL_VERSION_INFO >= (3, 2) CAN_IGNORE_UNKNOWN_OPENSSL_GROUPS = ssl.OPENSSL_VERSION_INFO >= (3, 3) CAN_GET_AVAILABLE_OPENSSL_GROUPS = ssl.OPENSSL_VERSION_INFO >= (3, 5) CAN_GET_AVAILABLE_OPENSSL_SIGALGS = ssl.OPENSSL_VERSION_INFO >= (3, 4) -CAN_SET_CLIENT_SIGALGS = "AWS-LC" not in ssl.OPENSSL_VERSION +CAN_SET_CLIENT_SIGALGS = not IS_AWS_LC CAN_IGNORE_UNKNOWN_OPENSSL_SIGALGS = ssl.OPENSSL_VERSION_INFO >= (3, 3) CAN_GET_SELECTED_OPENSSL_SIGALG = ssl.OPENSSL_VERSION_INFO >= (3, 5) PY_SSL_DEFAULT_CIPHERS = sysconfig.get_config_var('PY_SSL_DEFAULT_CIPHERS') @@ -383,6 +385,20 @@ def testing_context(server_cert=SIGNED_CERTFILE, *, server_chain=True, return client_context, server_context, hostname +def do_ssl_object_handshake(sslobject, outgoing, max_retry=25): + """Call do_handshake() on the sslobject and return the sent data. + + If do_handshake() fails more than *max_retry* times, return None. + """ + data, attempt = None, 0 + while not data and attempt < max_retry: + with contextlib.suppress(ssl.SSLWantReadError): + sslobject.do_handshake() + data = outgoing.read() + attempt += 1 + return data + + class BasicSocketTests(unittest.TestCase): def test_constants(self): @@ -395,7 +411,7 @@ def test_constants(self): ssl.OP_NO_COMPRESSION self.assertEqual(ssl.HAS_SNI, True) self.assertEqual(ssl.HAS_ECDH, True) - self.assertEqual(ssl.HAS_TLSv1_2, True) + self.assertIsInstance(ssl.HAS_TLSv1_2, bool) self.assertEqual(ssl.HAS_TLSv1_3, True) ssl.OP_NO_SSLv2 ssl.OP_NO_SSLv3 @@ -586,11 +602,11 @@ def test_openssl_version(self): # Some sanity checks follow # >= 1.1.1 self.assertGreaterEqual(n, 0x10101000) - # < 4.0 - self.assertLess(n, 0x40000000) + # < 5.0 + self.assertLess(n, 0x50000000) major, minor, fix, patch, status = t self.assertGreaterEqual(major, 1) - self.assertLess(major, 4) + self.assertLess(major, 5) self.assertGreaterEqual(minor, 0) self.assertLess(minor, 256) self.assertGreaterEqual(fix, 0) @@ -656,12 +672,14 @@ def test_openssl111_deprecations(self): ssl.OP_NO_TLSv1_2, ssl.OP_NO_TLSv1_3 ] - protocols = [ - ssl.PROTOCOL_TLSv1, - ssl.PROTOCOL_TLSv1_1, - ssl.PROTOCOL_TLSv1_2, - ssl.PROTOCOL_TLS - ] + protocols = [] + if hasattr(ssl, 'PROTOCOL_TLSv1'): + protocols.append(ssl.PROTOCOL_TLSv1) + if hasattr(ssl, 'PROTOCOL_TLSv1_1'): + protocols.append(ssl.PROTOCOL_TLSv1_1) + if hasattr(ssl, 'PROTOCOL_TLSv1_2'): + protocols.append(ssl.PROTOCOL_TLSv1_2) + protocols.append(ssl.PROTOCOL_TLS) versions = [ ssl.TLSVersion.SSLv3, ssl.TLSVersion.TLSv1, @@ -1205,6 +1223,7 @@ def test_min_max_version(self): ssl.TLSVersion.TLSv1, ssl.TLSVersion.TLSv1_1, ssl.TLSVersion.TLSv1_2, + ssl.TLSVersion.TLSv1_3, ssl.TLSVersion.SSLv3, } ) @@ -1218,7 +1237,7 @@ def test_min_max_version(self): with self.assertRaises(ValueError): ctx.minimum_version = 42 - if has_tls_protocol(ssl.PROTOCOL_TLSv1_1): + if has_tls_protocol('PROTOCOL_TLSv1_1'): ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_1) self.assertIn( @@ -1532,6 +1551,49 @@ def dummycallback(sock, servername, ctx): ctx.set_servername_callback(None) ctx.set_servername_callback(dummycallback) + def test_sni_callback_on_dead_references(self): + # See https://github.com/python/cpython/issues/146080. + c_ctx = make_test_context() + c_inc, c_out = ssl.MemoryBIO(), ssl.MemoryBIO() + client = c_ctx.wrap_bio(c_inc, c_out, server_hostname=SIGNED_CERTFILE_HOSTNAME) + + def sni_callback(sock, servername, ctx): pass + sni_callback = unittest.mock.Mock(wraps=sni_callback) + s_ctx = make_test_context(server_side=True, certfile=SIGNED_CERTFILE) + s_ctx.set_servername_callback(sni_callback) + + s_inc, s_out = ssl.MemoryBIO(), ssl.MemoryBIO() + server = s_ctx.wrap_bio(s_inc, s_out, server_side=True) + server_impl = server._sslobj + + # Perform the handshake on the client side first. + data = do_ssl_object_handshake(client, c_out) + sni_callback.assert_not_called() + if data is None: + self.skipTest("cannot establish a handshake from the client") + s_inc.write(data) + sni_callback.assert_not_called() + # Delete the server object before it starts doing its handshake + # and ensure that we did not call the SNI callback yet. + del server + gc.collect() + # Try to continue the server's handshake by directly using + # the internal SSL object. The latter is a weak reference + # stored in the server context and has now a dead owner. + with self.assertRaises(ssl.SSLError) as cm: + server_impl.do_handshake() + # The SNI C callback raised an exception before calling our callback. + sni_callback.assert_not_called() + + # In AWS-LC, any handshake failures reports SSL_R_PARSE_TLSEXT, + # while OpenSSL uses SSL_R_CALLBACK_FAILED on SNI callback failures. + if IS_AWS_LC: + libssl_error_reason = "PARSE_TLSEXT" + else: + libssl_error_reason = "callback failed" + self.assertIn(libssl_error_reason, str(cm.exception)) + self.assertEqual(cm.exception.errno, ssl.SSL_ERROR_SSL) + def test_sni_callback_refcycle(self): # Reference cycles through the servername callback are detected # and cleared. @@ -1675,23 +1737,24 @@ def test__create_stdlib_context(self): self.assertFalse(ctx.check_hostname) self._assert_context_options(ctx) - if has_tls_protocol(ssl.PROTOCOL_TLSv1): + if has_tls_protocol('PROTOCOL_TLSv1'): with warnings_helper.check_warnings(): ctx = ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1) self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1) self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) self._assert_context_options(ctx) - with warnings_helper.check_warnings(): - ctx = ssl._create_stdlib_context( - ssl.PROTOCOL_TLSv1_2, - cert_reqs=ssl.CERT_REQUIRED, - check_hostname=True - ) - self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1_2) - self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) - self.assertTrue(ctx.check_hostname) - self._assert_context_options(ctx) + if has_tls_protocol('PROTOCOL_TLSv1_2'): + with warnings_helper.check_warnings(): + ctx = ssl._create_stdlib_context( + ssl.PROTOCOL_TLSv1_2, + cert_reqs=ssl.CERT_REQUIRED, + check_hostname=True + ) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1_2) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + self._assert_context_options(ctx) ctx = ssl._create_stdlib_context(purpose=ssl.Purpose.CLIENT_AUTH) self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS_SERVER) @@ -3654,10 +3717,10 @@ def test_protocol_tlsv1_2(self): client_options=ssl.OP_NO_TLSv1_2) try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2') - if has_tls_protocol(ssl.PROTOCOL_TLSv1): + if has_tls_protocol('PROTOCOL_TLSv1'): try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1, False) try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_2, False) - if has_tls_protocol(ssl.PROTOCOL_TLSv1_1): + if has_tls_protocol('PROTOCOL_TLSv1_1'): try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_1, False) try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_2, False) diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index c7dc69defded50..6479676f155eca 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -397,6 +397,21 @@ def test_705836(self): big = (1 << 25) - 1 big = math.ldexp(big, 127 - 24) self.assertRaises(OverflowError, struct.pack, ">f", big) + self.assertRaises(OverflowError, struct.pack, "e", big) + unpacked = struct.unpack(">e", packed)[0] + self.assertEqual(big, unpacked) + big = (1 << 12) - 1 + big = math.ldexp(big, 15 - 11) + self.assertRaises(OverflowError, struct.pack, ">e", big) + self.assertRaises(OverflowError, struct.pack, "f", + "d", "d", + "e", "e", + ): + with self.subTest(format=format): + f = struct.unpack(format, struct.pack(format, 1.5))[0] + self.assertEqual(f, 1.5) + f = struct.unpack(format, struct.pack(format, NAN))[0] + self.assertTrue(math.isnan(f), f) + f = struct.unpack(format, struct.pack(format, INF))[0] + self.assertTrue(math.isinf(f), f) + self.assertEqual(math.copysign(1.0, f), 1.0) + f = struct.unpack(format, struct.pack(format, -INF))[0] + self.assertTrue(math.isinf(f), f) + self.assertEqual(math.copysign(1.0, f), -1.0) + class UnpackIteratorTest(unittest.TestCase): """ diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index a3129dbcb0a54e..d556f96bc532ed 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -569,16 +569,28 @@ def test_args_from_interpreter_flags(self): # -X options ['-X', 'dev'], ['-Wignore', '-X', 'dev'], + ['-X', 'cpu_count=4'], + ['-X', 'disable-remote-debug'], ['-X', 'faulthandler'], ['-X', 'importtime'], ['-X', 'importtime=2'], + ['-X', 'int_max_str_digits=1000'], + ['-X', 'lazy_imports=all'], + ['-X', 'no_debug_ranges'], ['-X', 'showrefcount'], ['-X', 'tracemalloc'], ['-X', 'tracemalloc=3'], + ['-X', 'warn_default_encoding'], ): with self.subTest(opts=opts): self.check_options(opts, 'args_from_interpreter_flags') + with os_helper.temp_dir() as temp_path: + prefix = os.path.join(temp_path, 'pycache') + opts = ['-X', f'pycache_prefix={prefix}'] + with self.subTest(opts=opts): + self.check_options(opts, 'args_from_interpreter_flags') + self.check_options(['-I', '-E', '-s', '-P'], 'args_from_interpreter_flags', ['-I']) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index e43f91eb9238f9..e6433f76a977b3 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -371,10 +371,12 @@ def test_get_platform(self): sys.platform = 'android' get_config_vars()['ANDROID_API_LEVEL'] = 9 for machine, abi in { - 'x86_64': 'x86_64', - 'i686': 'x86', 'aarch64': 'arm64_v8a', + 'arm': 'armeabi_v7a', 'armv7l': 'armeabi_v7a', + 'armv8l': 'armeabi_v7a', + 'i686': 'x86', + 'x86_64': 'x86_64', }.items(): with self.subTest(machine): self._set_uname(('Linux', 'localhost', '3.18.91+', @@ -579,10 +581,12 @@ def test_android_ext_suffix(self): machine = platform.machine() suffix = sysconfig.get_config_var('EXT_SUFFIX') expected_triplet = { - "x86_64": "x86_64-linux-android", - "i686": "i686-linux-android", "aarch64": "aarch64-linux-android", + "arm": "arm-linux-androideabi", "armv7l": "arm-linux-androideabi", + "armv8l": "arm-linux-androideabi", + "i686": "i686-linux-android", + "x86_64": "x86_64-linux-android", }[machine] self.assertEndsWith(suffix, f"-{expected_triplet}.so") diff --git a/Lib/test/test_timeit.py b/Lib/test/test_timeit.py index f8bc306b455a5d..8837e88ba638cf 100644 --- a/Lib/test/test_timeit.py +++ b/Lib/test/test_timeit.py @@ -364,10 +364,10 @@ def test_main_exception_fixed_reps(self): s = self.run_main(switches=['-n1', '1/0']) self.assert_exc_string(error_stringio.getvalue(), 'ZeroDivisionError') - def autorange(self, seconds_per_increment=1/1024, callback=None): + def autorange(self, seconds_per_increment=1/1024, callback=None, target_time=0.2): timer = FakeTimer(seconds_per_increment=seconds_per_increment) t = timeit.Timer(stmt=self.fake_stmt, setup=self.fake_setup, timer=timer) - return t.autorange(callback) + return t.autorange(callback, target_time=target_time) def test_autorange(self): num_loops, time_taken = self.autorange() @@ -379,6 +379,11 @@ def test_autorange_second(self): self.assertEqual(num_loops, 1) self.assertEqual(time_taken, 1.0) + def test_autorange_with_target_time(self): + num_loops, time_taken = self.autorange(target_time=1.0) + self.assertEqual(num_loops, 2000) + self.assertEqual(time_taken, 2000/1024) + def test_autorange_with_callback(self): def callback(a, b): print("{} {:.3f}".format(a, b)) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 7124e49b22e361..5dc11253e0d5c8 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -40,7 +40,7 @@ test_frame = namedtuple('frame', ['f_code', 'f_globals', 'f_locals']) test_tb = namedtuple('tb', ['tb_frame', 'tb_lineno', 'tb_next', 'tb_lasti']) -color_overrides = {"reset": "z", "filename": "fn", "error_highlight": "E"} +color_overrides = {"reset": "z", "filename": "fn", "error_highlight": "E", "note": "n"} colors = { color_overrides.get(k, k[0].lower()): v for k, v in _colorize.default_theme.traceback.items() @@ -5306,6 +5306,23 @@ def bar(): self.assertIn("return baz1(1,\n 2,3\n ,4)", lines) self.assertIn(red + "bar" + reset + boldr + "()" + reset, lines) + def test_colorized_exception_notes(self): + def foo(): + raise ValueError() + + try: + foo() + except Exception as e: + e.add_note("First note") + e.add_note("Second note") + exc = traceback.TracebackException.from_exception(e) + + lines = "".join(exc.format(colorize=True)) + note = colors["n"] + reset = colors["z"] + self.assertIn(note + "First note" + reset, lines) + self.assertIn(note + "Second note" + reset, lines) + def test_colorized_syntax_error(self): try: compile("a $ b", "", "exec") @@ -5314,7 +5331,7 @@ def test_colorized_syntax_error(self): e, capture_locals=True ) actual = "".join(exc.format(colorize=True)) - def expected(t, m, fn, l, f, E, e, z): + def expected(t, m, fn, l, f, E, e, z, n): return "".join( [ f' File {fn}""{z}, line {l}1{z}\n', @@ -5340,7 +5357,7 @@ def foo(): actual = tbstderr.getvalue().splitlines() lno_foo = foo.__code__.co_firstlineno - def expected(t, m, fn, l, f, E, e, z): + def expected(t, m, fn, l, f, E, e, z, n): return [ 'Traceback (most recent call last):', f' File {fn}"{__file__}"{z}, ' @@ -5373,7 +5390,7 @@ def foo(): lno_foo = foo.__code__.co_firstlineno actual = "".join(exc.format(colorize=True)).splitlines() - def expected(t, m, fn, l, f, E, e, z): + def expected(t, m, fn, l, f, E, e, z, n): return [ f" + Exception Group Traceback (most recent call last):", f' | File {fn}"{__file__}"{z}, line {l}{lno_foo+9}{z}, in {f}test_colorized_traceback_from_exception_group{z}', diff --git a/Lib/test/test_unpack_ex.py b/Lib/test/test_unpack_ex.py index 904cf4f626ae78..33c96b84964b59 100644 --- a/Lib/test/test_unpack_ex.py +++ b/Lib/test/test_unpack_ex.py @@ -134,6 +134,22 @@ ... TypeError: 'list' object is not a mapping + >>> class OnlyKeys: + ... def keys(self): + ... return ['key'] + >>> {**OnlyKeys()} + Traceback (most recent call last): + ... + TypeError: 'OnlyKeys' object is not subscriptable + + >>> class BrokenKeys: + ... def keys(self): + ... return 1 + >>> {**BrokenKeys()} + Traceback (most recent call last): + ... + TypeError: test.test_unpack_ex.BrokenKeys.keys() must return an iterable, not int + >>> len(eval("{" + ", ".join("**{{{}: {}}}".format(i, i) ... for i in range(1000)) + "}")) 1000 @@ -560,6 +576,176 @@ """ +def test_errors_in_iter(): + """ + >>> class A: + ... def __iter__(self): + ... raise exc + ... + >>> exc = ZeroDivisionError('some error') + >>> [*A()] + Traceback (most recent call last): + ... + ZeroDivisionError: some error + + >>> {*A()} + Traceback (most recent call last): + ... + ZeroDivisionError: some error + + >>> exc = AttributeError('some error') + >>> [*A()] + Traceback (most recent call last): + ... + AttributeError: some error + + >>> {*A()} + Traceback (most recent call last): + ... + AttributeError: some error + + >>> exc = TypeError('some error') + >>> [*A()] + Traceback (most recent call last): + ... + TypeError: some error + + >>> {*A()} + Traceback (most recent call last): + ... + TypeError: some error + """ + +def test_errors_in_next(): + """ + >>> class I: + ... def __iter__(self): + ... return self + ... def __next__(self): + ... raise exc + ... + >>> class A: + ... def __iter__(self): + ... return I() + ... + + >>> exc = ZeroDivisionError('some error') + >>> [*A()] + Traceback (most recent call last): + ... + ZeroDivisionError: some error + + >>> {*A()} + Traceback (most recent call last): + ... + ZeroDivisionError: some error + + >>> exc = AttributeError('some error') + >>> [*A()] + Traceback (most recent call last): + ... + AttributeError: some error + + >>> {*A()} + Traceback (most recent call last): + ... + AttributeError: some error + + >>> exc = TypeError('some error') + >>> [*A()] + Traceback (most recent call last): + ... + TypeError: some error + + >>> {*A()} + Traceback (most recent call last): + ... + TypeError: some error + """ + +def test_errors_in_keys(): + """ + >>> class D: + ... def keys(self): + ... raise exc + ... + >>> exc = ZeroDivisionError('some error') + >>> {**D()} + Traceback (most recent call last): + ... + ZeroDivisionError: some error + + >>> exc = AttributeError('some error') + >>> {**D()} + Traceback (most recent call last): + ... + AttributeError: some error + + >>> exc = KeyError('some error') + >>> {**D()} + Traceback (most recent call last): + ... + KeyError: 'some error' + """ + +def test_errors_in_keys_next(): + """ + >>> class I: + ... def __iter__(self): + ... return self + ... def __next__(self): + ... raise exc + ... + >>> class D: + ... def keys(self): + ... return I() + ... + >>> exc = ZeroDivisionError('some error') + >>> {**D()} + Traceback (most recent call last): + ... + ZeroDivisionError: some error + + >>> exc = AttributeError('some error') + >>> {**D()} + Traceback (most recent call last): + ... + AttributeError: some error + + >>> exc = KeyError('some error') + >>> {**D()} + Traceback (most recent call last): + ... + KeyError: 'some error' + """ + +def test_errors_in_getitem(): + """ + >>> class D: + ... def keys(self): + ... return ['key'] + ... def __getitem__(self, key): + ... raise exc + ... + >>> exc = ZeroDivisionError('some error') + >>> {**D()} + Traceback (most recent call last): + ... + ZeroDivisionError: some error + + >>> exc = AttributeError('some error') + >>> {**D()} + Traceback (most recent call last): + ... + AttributeError: some error + + >>> exc = KeyError('some error') + >>> {**D()} + Traceback (most recent call last): + ... + KeyError: 'some error' + """ + __test__ = {'doctests' : doctests} def load_tests(loader, tests, pattern): diff --git a/Lib/test/test_userdict.py b/Lib/test/test_userdict.py index 75de9ea252de98..c60135ca5a12a8 100644 --- a/Lib/test/test_userdict.py +++ b/Lib/test/test_userdict.py @@ -1,8 +1,18 @@ # Check every path through every method of UserDict +from collections import UserDict from test import mapping_tests import unittest import collections +import types + + +class UserDictSubclass(UserDict): + pass + +class UserDictSubclass2(UserDict): + pass + d0 = {} d1 = {"one": 1} @@ -155,6 +165,25 @@ def test_init(self): self.assertRaises(TypeError, collections.UserDict, (), ()) self.assertRaises(TypeError, collections.UserDict.__init__) + def test_data(self): + u = UserDict() + self.assertEqual(u.data, {}) + self.assertIs(type(u.data), dict) + d = {'a': 1, 'b': 2} + u = UserDict(d) + self.assertEqual(u.data, d) + self.assertIsNot(u.data, d) + self.assertIs(type(u.data), dict) + u = UserDict(u) + self.assertEqual(u.data, d) + self.assertIs(type(u.data), dict) + u = UserDict([('a', 1), ('b', 2)]) + self.assertEqual(u.data, d) + self.assertIs(type(u.data), dict) + u = UserDict(a=1, b=2) + self.assertEqual(u.data, d) + self.assertIs(type(u.data), dict) + def test_update(self): for kw in 'self', 'dict', 'other', 'iterable': d = collections.UserDict() @@ -215,6 +244,69 @@ class G(collections.UserDict): test_repr_deep = mapping_tests.TestHashMappingProtocol.test_repr_deep + def test_mixed_or(self): + for t in UserDict, dict, frozendict, types.MappingProxyType: + with self.subTest(t.__name__): + u = UserDict({0: 'a', 1: 'b'}) | t({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDict) + + u = t({0: 'a', 1: 'b'}) | UserDict({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDict) + + u = UserDict({0: 'a', 1: 'b'}) | UserDictSubclass({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDict) + + u = UserDictSubclass({0: 'a', 1: 'b'}) | UserDict({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDictSubclass) + + u = UserDictSubclass({0: 'a', 1: 'b'}) | UserDictSubclass2({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDictSubclass) + + u = UserDict({1: 'c', 2: 'd'}).__ror__(UserDict({0: 'a', 1: 'b'})) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDict) + + u = UserDictSubclass({1: 'c', 2: 'd'}).__ror__(UserDictSubclass2({0: 'a', 1: 'b'})) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDictSubclass) + + def test_mixed_ior(self): + for t in UserDict, dict, frozendict, types.MappingProxyType: + with self.subTest(t.__name__): + u = u2 = UserDict({0: 'a', 1: 'b'}) + u |= t({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDict) + self.assertIs(u, u2) + + u = dict({0: 'a', 1: 'b'}) + u |= UserDict({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), dict) + + u = u2 = UserDict({0: 'a', 1: 'b'}) + u |= UserDictSubclass({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDict) + self.assertIs(u, u2) + + u = u2 = UserDictSubclass({0: 'a', 1: 'b'}) + u |= UserDict({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDictSubclass) + self.assertIs(u, u2) + + u = u2 = UserDictSubclass({0: 'a', 1: 'b'}) + u |= UserDictSubclass2({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDictSubclass) + self.assertIs(u, u2) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_userlist.py b/Lib/test/test_userlist.py index d3d9f4cff8da3a..3e5c5ff19458eb 100644 --- a/Lib/test/test_userlist.py +++ b/Lib/test/test_userlist.py @@ -2,12 +2,36 @@ from collections import UserList from test import list_tests +from test import support import unittest +class UserListSubclass(UserList): + pass + +class UserListSubclass2(UserList): + pass + + class UserListTest(list_tests.CommonTest): type2test = UserList + def test_data(self): + u = UserList() + self.assertEqual(u.data, []) + self.assertIs(type(u.data), list) + a = [1, 2] + u = UserList(a) + self.assertEqual(u.data, a) + self.assertIsNot(u.data, a) + self.assertIs(type(u.data), list) + u = UserList(u) + self.assertEqual(u.data, a) + self.assertIs(type(u.data), list) + u = UserList("spam") + self.assertEqual(u.data, list("spam")) + self.assertIs(type(u.data), list) + def test_getslice(self): super().test_getslice() l = [0, 1, 2, 3, 4] @@ -24,34 +48,74 @@ def test_slice_type(self): self.assertIsInstance(u[:], u.__class__) self.assertEqual(u[:],u) - def test_add_specials(self): - u = UserList("spam") - u2 = u + "eggs" - self.assertEqual(u2, list("spameggs")) + def test_mixed_add(self): + for t in UserList, list, str, tuple, iter: + with self.subTest(t.__name__): + u = UserList("spam") + t("eggs") + self.assertEqual(u, list("spameggs")) + self.assertIs(type(u), UserList) + + u = t("spam") + UserList("eggs") + self.assertEqual(u, list("spameggs")) + self.assertIs(type(u), UserList) + + u = UserList("spam") + UserListSubclass("eggs") + self.assertEqual(u, list("spameggs")) + self.assertIs(type(u), UserList) - def test_radd_specials(self): - u = UserList("eggs") - u2 = "spam" + u + u = UserListSubclass("spam") + UserList("eggs") + self.assertEqual(u, list("spameggs")) + self.assertIs(type(u), UserListSubclass) + + u = UserListSubclass("spam") + UserListSubclass2("eggs") + self.assertEqual(u, list("spameggs")) + self.assertIs(type(u), UserListSubclass) + + u2 = UserList("eggs").__radd__(UserList("spam")) self.assertEqual(u2, list("spameggs")) - u2 = u.__radd__(UserList("spam")) + self.assertIs(type(u), UserListSubclass) + + u2 = UserListSubclass("eggs").__radd__(UserListSubclass2("spam")) self.assertEqual(u2, list("spameggs")) + self.assertIs(type(u), UserListSubclass) - def test_iadd(self): - super().test_iadd() - u = [0, 1] - u += UserList([0, 1]) - self.assertEqual(u, [0, 1, 0, 1]) + def test_mixed_iadd(self): + for t in UserList, list, str, tuple, iter: + with self.subTest(t.__name__): + u = u2 = UserList("spam") + u += t("eggs") + self.assertEqual(u, list("spameggs")) + self.assertIs(type(u), UserList) + self.assertIs(u, u2) - def test_mixedcmp(self): - u = self.type2test([0, 1]) - self.assertEqual(u, [0, 1]) - self.assertNotEqual(u, [0]) - self.assertNotEqual(u, [0, 2]) + u = t("spam") + u += UserList("eggs") + self.assertEqual(u, list("spameggs")) + self.assertIs(type(u), UserList) - def test_mixedadd(self): + u = u2 = UserList("spam") + u += UserListSubclass("eggs") + self.assertEqual(u, list("spameggs")) + self.assertIs(type(u), UserList) + self.assertIs(u, u2) + + u = u2 = UserListSubclass("spam") + u += UserList("eggs") + self.assertEqual(u, list("spameggs")) + self.assertIs(type(u), UserListSubclass) + self.assertIs(u, u2) + + u = u2 = UserListSubclass("spam") + u += UserListSubclass2("eggs") + self.assertEqual(u, list("spameggs")) + self.assertIs(type(u), UserListSubclass) + self.assertIs(u, u2) + + def test_mixed_cmp(self): u = self.type2test([0, 1]) - self.assertEqual(u + [], u) - self.assertEqual(u + [2], [0, 1, 2]) + self._assert_cmp(u, [0, 1], 0) + self._assert_cmp(u, [0], 1) + self._assert_cmp(u, [0, 2], -1) def test_getitemoverwriteiter(self): # Verify that __getitem__ overrides *are* recognized by __iter__ @@ -60,6 +124,43 @@ def __getitem__(self, key): return str(key) + '!!!' self.assertEqual(next(iter(T((1,2)))), "0!!!") + def test_implementation(self): + u = UserList([1]) + with (support.swap_attr(UserList, '__len__', None), + support.swap_attr(UserList, 'insert', None)): + u.append(2) + self.assertEqual(u, [1, 2]) + with support.swap_attr(UserList, 'append', None): + u.extend([3, 4]) + self.assertEqual(u, [1, 2, 3, 4]) + with support.swap_attr(UserList, 'append', None): + u.extend(UserList([3, 4])) + self.assertEqual(u, [1, 2, 3, 4, 3, 4]) + with support.swap_attr(UserList, '__iter__', None): + c = u.count(3) + self.assertEqual(c, 2) + with (support.swap_attr(UserList, '__iter__', None), + support.swap_attr(UserList, '__getitem__', None)): + i = u.index(4) + self.assertEqual(i, 3) + with (support.swap_attr(UserList, 'index', None), + support.swap_attr(UserList, '__getitem__', None)): + u.remove(3) + self.assertEqual(u, [1, 2, 4, 3, 4]) + with (support.swap_attr(UserList, '__getitem__', None), + support.swap_attr(UserList, '__delitem__', None)): + u.pop() + self.assertEqual(u, [1, 2, 4, 3]) + with (support.swap_attr(UserList, '__len__', None), + support.swap_attr(UserList, '__getitem__', None), + support.swap_attr(UserList, '__setitem__', None)): + u.reverse() + self.assertEqual(u, [3, 4, 2, 1]) + with (support.swap_attr(UserList, '__len__', None), + support.swap_attr(UserList, 'pop', None)): + u.clear() + self.assertEqual(u, []) + def test_userlist_copy(self): u = self.type2test([6, 8, 1, 9, 1]) v = u.copy() diff --git a/Lib/test/test_userstring.py b/Lib/test/test_userstring.py index 74df52f5412af0..cc85c06bf93363 100644 --- a/Lib/test/test_userstring.py +++ b/Lib/test/test_userstring.py @@ -3,9 +3,18 @@ import unittest from test import string_tests +from test import support from collections import UserString + +class UserStringSubclass(UserString): + pass + +class UserStringSubclass2(UserString): + pass + + class UserStringTest( string_tests.StringLikeTest, unittest.TestCase @@ -40,6 +49,78 @@ def checkcall(self, object, methodname, *args): # we don't fix the arguments, because UserString can't cope with it getattr(object, methodname)(*args) + def test_data(self): + u = UserString("spam") + self.assertEqual(u.data, "spam") + self.assertIs(type(u.data), str) + u = UserString(u) + self.assertEqual(u.data, "spam") + self.assertIs(type(u.data), str) + u = UserString(42) + self.assertEqual(u.data, "42") + self.assertIs(type(u.data), str) + + def test_mixed_add(self): + u = UserString("spam") + "eggs" + self.assertEqual(u, "spameggs") + self.assertIs(type(u), UserString) + + u = "spam" + UserString("eggs") + self.assertEqual(u, "spameggs") + self.assertIs(type(u), UserString) + + u = UserString("spam") + UserStringSubclass("eggs") + self.assertEqual(u, "spameggs") + self.assertIs(type(u), UserString) + + u = UserStringSubclass("spam") + UserString("eggs") + self.assertEqual(u, "spameggs") + self.assertIs(type(u), UserStringSubclass) + + u = UserStringSubclass("spam") + UserStringSubclass2("eggs") + self.assertEqual(u, "spameggs") + self.assertIs(type(u), UserStringSubclass) + + u2 = UserString("eggs").__radd__(UserString("spam")) + self.assertEqual(u2, "spameggs") + self.assertIs(type(u), UserStringSubclass) + + u2 = UserStringSubclass("eggs").__radd__(UserStringSubclass2("spam")) + self.assertEqual(u2, "spameggs") + self.assertIs(type(u), UserStringSubclass) + + def test_mixed_iadd(self): + u = UserString("spam") + u += "eggs" + self.assertEqual(u, "spameggs") + self.assertIs(type(u), UserString) + + u = "spam" + u += UserString("eggs") + self.assertEqual(u, "spameggs") + self.assertIs(type(u), UserString) + + u = UserString("spam") + u += UserStringSubclass("eggs") + self.assertEqual(u, "spameggs") + self.assertIs(type(u), UserString) + + u = UserStringSubclass("spam") + u += UserString("eggs") + self.assertEqual(u, "spameggs") + self.assertIs(type(u), UserStringSubclass) + + u = UserStringSubclass("spam") + u += UserStringSubclass2("eggs") + self.assertEqual(u, "spameggs") + self.assertIs(type(u), UserStringSubclass) + + def test_mixed_cmp(self): + a = self.fixtype('ab') + self._assert_cmp(a, 'ab', 0) + self._assert_cmp(a, 'a', 1) + self._assert_cmp(a, 'ac', -1) + def test_rmod(self): class ustr2(UserString): pass @@ -66,6 +147,20 @@ def test_encode_explicit_none_args(self): # Check that errors defaults to 'strict' self.checkraises(UnicodeError, '\ud800', 'encode', None, None) + def test_implementation(self): + s = UserString('ababahalamaha') + with support.swap_attr(UserString, '__iter__', None): + c = s.count('a') + c2 = s.count(UserString('a')) + self.assertEqual(c, 7) + self.assertEqual(c2, 7) + with (support.swap_attr(UserString, '__iter__', None), + support.swap_attr(UserString, '__getitem__', None)): + i = s.index('h') + i2 = s.index(UserString('h')) + self.assertEqual(i, 5) + self.assertEqual(i2, 5) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index a6af5057cc8968..d86844c1a29a9a 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -1,5 +1,6 @@ from contextlib import contextmanager import linecache +import logging import os import importlib import inspect @@ -509,6 +510,47 @@ def test_catchwarnings_with_simplefilter_error(self): stderr = stderr.getvalue() self.assertIn(error_msg, stderr) + def test_catchwarnings_with_showwarning(self): + # gh-146358: catch_warnings must override warnings.showwarning() + # if it's not the default implementation. + + warns = [] + def custom_showwarning(message, category, filename, lineno, + file=None, line=None): + warns.append(message) + + with self.module.catch_warnings(): + self.module.resetwarnings() + + with support.swap_attr(self.module, 'showwarning', + custom_showwarning): + with self.module.catch_warnings(record=True) as recorded: + self.module.warn("recorded") + self.assertEqual(len(recorded), 1) + self.assertEqual(str(recorded[0].message), 'recorded') + self.assertIs(self.module.showwarning, custom_showwarning) + + self.module.warn("custom") + + self.assertEqual(len(warns), 1) + self.assertEqual(str(warns[0]), "custom") + + def test_catchwarnings_logging(self): + # gh-146358: catch_warnings(record=True) must replace the + # showwarning() function set by logging.captureWarnings(True). + + with self.module.catch_warnings(): + self.module.resetwarnings() + logging.captureWarnings(True) + + with self.module.catch_warnings(record=True) as recorded: + self.module.warn("recorded") + self.assertEqual(len(recorded), 1) + self.assertEqual(str(recorded[0].message), 'recorded') + + logging.captureWarnings(False) + + class CFilterTests(FilterTests, unittest.TestCase): module = c_warnings diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index d5bb1400d2717a..ea161ea1a43ea5 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -57,6 +57,14 @@ def _test(self, meth, *, args=[URL], kw={}, options, arguments): popen_args.pop(popen_args.index(option)) self.assertEqual(popen_args, arguments) + def test_reject_dash_prefixes(self): + browser = self.browser_class(name=CMD_NAME) + with self.assertRaisesRegex( + ValueError, + r"^Invalid URL \(leading dash disallowed\): '--key=val http.*'$" + ): + browser.open(f"--key=val {URL}") + class GenericBrowserCommandTest(CommandTestMixin, unittest.TestCase): @@ -67,11 +75,6 @@ def test_open(self): options=[], arguments=[URL]) - def test_reject_dash_prefixes(self): - browser = self.browser_class(name=CMD_NAME) - with self.assertRaises(ValueError): - browser.open(f"--key=val {URL}") - class BackgroundBrowserCommandTest(CommandTestMixin, unittest.TestCase): @@ -326,7 +329,6 @@ def close(self): @unittest.skipUnless(sys.platform == "darwin", "macOS specific test") @requires_subprocess() class MacOSXOSAScriptTest(unittest.TestCase): - def setUp(self): # Ensure that 'BROWSER' is not set to 'open' or something else. # See: https://github.com/python/cpython/issues/131254. @@ -376,6 +378,13 @@ def test_explicit_browser(self): self.assertIn('tell application "safari"', script) self.assertIn('open location "https://python.org"', script) + def test_reject_dash_prefixes(self): + with self.assertRaisesRegex( + ValueError, + r"^Invalid URL \(leading dash disallowed\): '--key=val http.*'$" + ): + self.browser.open(f"--key=val {URL}") + class BrowserRegistrationTest(unittest.TestCase): diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 5b06e422672b1d..b380d0276b0169 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -381,6 +381,19 @@ def test_simpleops(self): self.serialize_check(element, '') + def test_positional_only_parameter(self): + # Test Element positional-only parameters (gh-144846). + + # 'tag' is positional-only + with self.assertRaises(TypeError): + ET.Element(tag='fail') + + # 'tag' and 'attrib' as kwarg/attribute names + e = ET.Element('e', attrib={'attrib': 'foo'}, tag='bar') + self.assertEqual(e.tag, 'e') + self.assertEqual(e.get('attrib'), 'foo') + self.assertEqual(e.get('tag'), 'bar') + def test_cdata(self): # Test CDATA handling (etc). @@ -484,6 +497,28 @@ def test_attrib(self): self.assertEqual(ET.tostring(elem), b'') + def test_subelement_positional_only_parameter(self): + # Test SubElement positional-only parameters (gh-144270). + parent = ET.Element('parent') + + # 'parent' and 'tag' are positional-only + with self.assertRaises(TypeError): + ET.SubElement(parent=parent, tag='fail') + with self.assertRaises(TypeError): + ET.SubElement(parent, tag='fail') + + # 'attrib' can be passed as keyword + sub1 = ET.SubElement(parent, 'sub1', attrib={'key': 'value'}) + self.assertEqual(sub1.get('key'), 'value') + + # 'tag' and 'parent' as kwargs become XML attributes, not func params + sub2 = ET.SubElement(parent, 'sub2', attrib={'attrib': 'foo'}, + tag='bar', parent='baz') + self.assertEqual(sub2.tag, 'sub2') + self.assertEqual(sub2.get('attrib'), 'foo') + self.assertEqual(sub2.get('tag'), 'bar') + self.assertEqual(sub2.get('parent'), 'baz') + def test_makeelement(self): # Test makeelement handling. diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index aaab4709464fd0..7502b120825fbc 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -741,6 +741,38 @@ def test_empty_zone(self): with self.assertRaises(ValueError): self.klass.from_file(zf) + def test_invalid_transition_index(self): + STD = ZoneOffset("STD", ZERO) + DST = ZoneOffset("DST", ONE_H, ONE_H) + + zf = self.construct_zone([ + ZoneTransition(datetime(2026, 3, 1, 2), STD, DST), + ZoneTransition(datetime(2026, 11, 1, 2), DST, STD), + ], after="", version=1) + + data = bytearray(zf.read()) + timecnt = struct.unpack_from(">l", data, 32)[0] + idx_offset = 44 + timecnt * 4 + data[idx_offset + 1] = 2 # typecnt is 2, so index 2 is OOB + f = io.BytesIO(bytes(data)) + + with self.assertRaises(ValueError): + self.klass.from_file(f) + + def test_transition_lookahead_out_of_bounds(self): + STD = ZoneOffset("STD", ZERO) + DST = ZoneOffset("DST", ONE_H, ONE_H) + EXT = ZoneOffset("EXT", ONE_H) + + zf = self.construct_zone([ + ZoneTransition(datetime(2026, 3, 1), STD, DST), + ZoneTransition(datetime(2026, 6, 1), DST, EXT), + ZoneTransition(datetime(2026, 9, 1), EXT, DST), + ], after="") + + zi = self.klass.from_file(zf) + self.assertIsNotNone(zi) + def test_zone_very_large_timestamp(self): """Test when a transition is in the far past or future. diff --git a/Lib/timeit.py b/Lib/timeit.py index 80791acdeca23f..f900da6ffe7d67 100644 --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -7,7 +7,7 @@ Library usage: see the Timer class. Command line usage: - python timeit.py [-n N] [-r N] [-s S] [-p] [-h] [--] [statement] + python timeit.py [-n N] [-r N] [-s S] [-p] [-h] [-t T] [--] [statement] Options: -n/--number N: how many times to execute 'statement' (default: see below) @@ -17,6 +17,9 @@ -p/--process: use time.process_time() (default is time.perf_counter()) -v/--verbose: print raw timing results; repeat for more digits precision -u/--unit: set the output time unit (nsec, usec, msec, or sec) + -t/--target-time T: if --number is 0 the code will run until it + takes *at least* this many seconds + (default: 0.2) -h/--help: print this usage message and exit --: separate options from statement, use when statement starts with - statement: statement to be timed (default 'pass') @@ -28,7 +31,7 @@ If -n is not given, a suitable number of loops is calculated by trying increasing numbers from the sequence 1, 2, 5, 10, 20, 50, ... until the -total time is at least 0.2 seconds. +total time is at least --target-time seconds. Note: there is a certain baseline overhead associated with executing a pass statement. It differs between versions. The code here doesn't try @@ -57,6 +60,7 @@ default_number = 1000000 default_repeat = 5 default_timer = time.perf_counter +default_target_time = 0.2 _globals = globals @@ -212,12 +216,13 @@ def repeat(self, repeat=default_repeat, number=default_number): r.append(t) return r - def autorange(self, callback=None): - """Return the number of loops and time taken so that total time >= 0.2. + def autorange(self, callback=None, target_time=default_target_time): + """Return the number of loops and time taken so that + total time >= target_time (default is 0.2 seconds). Calls the timeit method with increasing numbers from the sequence - 1, 2, 5, 10, 20, 50, ... until the time taken is at least 0.2 - second. Returns (number, time_taken). + 1, 2, 5, 10, 20, 50, ... until the target time is reached. + Returns (number, time_taken). If *callback* is given and is not None, it will be called after each trial with two arguments: ``callback(number, time_taken)``. @@ -229,7 +234,7 @@ def autorange(self, callback=None): time_taken = self.timeit(number) if callback: callback(number, time_taken) - if time_taken >= 0.2: + if time_taken >= target_time: return (number, time_taken) i *= 10 @@ -270,9 +275,10 @@ def main(args=None, *, _wrap_timer=None): colorize = _colorize.can_colorize() try: - opts, args = getopt.getopt(args, "n:u:s:r:pvh", + opts, args = getopt.getopt(args, "n:u:s:r:pt:vh", ["number=", "setup=", "repeat=", - "process", "verbose", "unit=", "help"]) + "process", "target-time=", + "verbose", "unit=", "help"]) except getopt.error as err: print(err) print("use -h/--help for command line help") @@ -281,6 +287,7 @@ def main(args=None, *, _wrap_timer=None): timer = default_timer stmt = "\n".join(args) or "pass" number = 0 # auto-determine + target_time = default_target_time setup = [] repeat = default_repeat verbose = 0 @@ -305,6 +312,8 @@ def main(args=None, *, _wrap_timer=None): repeat = 1 if o in ("-p", "--process"): timer = time.process_time + if o in ("-t", "--target-time"): + target_time = float(a) if o in ("-v", "--verbose"): if verbose: precision += 1 @@ -324,7 +333,7 @@ def main(args=None, *, _wrap_timer=None): t = Timer(stmt, setup, timer) if number == 0: - # determine number so that 0.2 <= total time < 2.0 + # determine number so that total time >= target_time callback = None if verbose: def callback(number, time_taken): @@ -333,7 +342,7 @@ def callback(number, time_taken): print(msg.format(num=number, s='s' if plural else '', secs=time_taken, prec=precision)) try: - number, _ = t.autorange(callback) + number, _ = t.autorange(callback, target_time) except: t.print_exc(colorize=colorize) return 1 diff --git a/Lib/traceback.py b/Lib/traceback.py index 56a72ce7f5b293..1f9f151ebf5d39 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -993,6 +993,10 @@ def _display_width(line, offset=None): ) +def _format_note(note, indent, theme): + for l in note.split("\n"): + yield f"{indent}{theme.note}{l}{theme.reset}\n" + class _ExceptionPrintContext: def __init__(self): @@ -1291,6 +1295,10 @@ def format_exception_only(self, *, show_group=False, _depth=0, **kwargs): well, recursively, with indentation relative to their nesting depth. """ colorize = kwargs.get("colorize", False) + if colorize: + theme = _colorize.get_theme(force_color=True).traceback + else: + theme = _colorize.get_theme(force_no_color=True).traceback indent = 3 * _depth * ' ' if not self._have_exc_type: @@ -1319,9 +1327,10 @@ def format_exception_only(self, *, show_group=False, _depth=0, **kwargs): ): for note in self.__notes__: note = _safe_string(note, 'note') - yield from [indent + l + '\n' for l in note.split('\n')] + yield from _format_note(note, indent, theme) elif self.__notes__ is not None: - yield indent + "{}\n".format(_safe_string(self.__notes__, '__notes__', func=repr)) + note = _safe_string(self.__notes__, '__notes__', func=repr) + yield from _format_note(note, indent, theme) if self.exceptions and show_group: for ex in self.exceptions: diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index 9ead2990e818e5..deb6e64d17421b 100644 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -167,7 +167,7 @@ def open_new_tab(self, url): def _check_url(url): """Ensures that the URL is safe to pass to subprocesses as a parameter""" if url and url.lstrip().startswith("-"): - raise ValueError(f"Invalid URL: {url}") + raise ValueError(f"Invalid URL (leading dash disallowed): {url!r}") class GenericBrowser(BaseBrowser): diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 57c5b64ea3ba70..85766e02b531ce 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -164,7 +164,7 @@ class Element: """ - def __init__(self, tag, attrib={}, **extra): + def __init__(self, tag, /, attrib={}, **extra): if not isinstance(attrib, (dict, frozendict)): raise TypeError("attrib must be dict or frozendict, not %s" % ( attrib.__class__.__name__,)) @@ -416,7 +416,7 @@ def itertext(self): yield t -def SubElement(parent, tag, attrib={}, **extra): +def SubElement(parent, tag, /, attrib={}, **extra): """Subelement factory which creates an element instance, and appends it to an existing parent. diff --git a/Lib/zoneinfo/_common.py b/Lib/zoneinfo/_common.py index 59f3f0ce853f74..98668c15d8bf94 100644 --- a/Lib/zoneinfo/_common.py +++ b/Lib/zoneinfo/_common.py @@ -67,6 +67,10 @@ def load_data(fobj): f">{timecnt}{time_type}", fobj.read(timecnt * time_size) ) trans_idx = struct.unpack(f">{timecnt}B", fobj.read(timecnt)) + + if max(trans_idx) >= typecnt: + raise ValueError("Invalid transition index found while reading TZif: " + f"{max(trans_idx)}") else: trans_list_utc = () trans_idx = () diff --git a/Lib/zoneinfo/_zoneinfo.py b/Lib/zoneinfo/_zoneinfo.py index bd3fefc6c9d959..7063eb6a9025ac 100644 --- a/Lib/zoneinfo/_zoneinfo.py +++ b/Lib/zoneinfo/_zoneinfo.py @@ -338,7 +338,7 @@ def _utcoff_to_dstoff(trans_idx, utcoffsets, isdsts): if not isdsts[comp_idx]: dstoff = utcoff - utcoffsets[comp_idx] - if not dstoff and idx < (typecnt - 1): + if not dstoff and idx < (typecnt - 1) and i + 1 < len(trans_idx): comp_idx = trans_idx[i + 1] # If the following transition is also DST and we couldn't diff --git a/Makefile.pre.in b/Makefile.pre.in index 5ea00537629de0..354580aa482d25 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2358,10 +2358,10 @@ testios: fi # Clone the testbed project into the XCFOLDER - $(PYTHON_FOR_BUILD) $(srcdir)/Apple/testbed clone --framework $(PYTHONFRAMEWORKPREFIX) "$(XCFOLDER)" + $(PYTHON_FOR_BUILD) $(srcdir)/Platforms/Apple/testbed clone --framework $(PYTHONFRAMEWORKPREFIX) "$(XCFOLDER)" # Run the testbed project - $(PYTHON_FOR_BUILD) "$(XCFOLDER)" run --verbose -- test -uall --single-process --rerun -W + $(PYTHON_FOR_BUILD) "$(XCFOLDER)" run --verbose -- test -uall --single-process --rerun -W --pythoninfo # Like test, but using --slow-ci which enables all test resources and use # longer timeout. Run an optional pybuildbot.identify script to include @@ -3286,10 +3286,10 @@ clean-retain-profile: pycremoval -find build -type f -a ! -name '*.gc??' -exec rm -f {} ';' -rm -f Include/pydtrace_probes.h -rm -f profile-gen-stamp - -rm -rf Apple/iOS/testbed/Python.xcframework/ios-*/bin - -rm -rf Apple/iOS/testbed/Python.xcframework/ios-*/lib - -rm -rf Apple/iOS/testbed/Python.xcframework/ios-*/include - -rm -rf Apple/iOS/testbed/Python.xcframework/ios-*/Python.framework + -rm -rf Platforms/Apple/iOS/testbed/Python.xcframework/ios-*/bin + -rm -rf Platforms/Apple/iOS/testbed/Python.xcframework/ios-*/lib + -rm -rf Platforms/Apple/iOS/testbed/Python.xcframework/ios-*/include + -rm -rf Platforms/Apple/iOS/testbed/Python.xcframework/ios-*/Python.framework .PHONY: profile-removal profile-removal: @@ -3323,7 +3323,7 @@ clobber: clean config.cache config.log pyconfig.h Modules/config.c -rm -rf build platform -rm -rf $(PYTHONFRAMEWORKDIR) - -rm -rf Apple/iOS/Frameworks + -rm -rf Platforms/Apple/iOS/Frameworks -rm -rf iOSTestbed.* -rm -f python-config.py python-config -rm -rf cross-build diff --git a/Misc/NEWS.d/next/Build/2026-03-10-16-58-55.gh-issue-138850.CkqTw6.rst b/Misc/NEWS.d/next/Build/2026-03-10-16-58-55.gh-issue-138850.CkqTw6.rst new file mode 100644 index 00000000000000..256f13b28772cc --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-03-10-16-58-55.gh-issue-138850.CkqTw6.rst @@ -0,0 +1 @@ +Add :option:`--disable-epoll` to ``configure`` diff --git a/Misc/NEWS.d/next/Build/2026-03-23-20-06-35.gh-issue-146210.C01Rmq.rst b/Misc/NEWS.d/next/Build/2026-03-23-20-06-35.gh-issue-146210.C01Rmq.rst new file mode 100644 index 00000000000000..ce59a9a3a571b4 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-03-23-20-06-35.gh-issue-146210.C01Rmq.rst @@ -0,0 +1,2 @@ +Fix building the jit stencils on Windows when the interpreter is built with +a different clang version. Patch by Chris Eibl. diff --git a/Misc/NEWS.d/next/Build/2026-03-26-12-27-42.gh-issue-146444.JKJuEa.rst b/Misc/NEWS.d/next/Build/2026-03-26-12-27-42.gh-issue-146444.JKJuEa.rst new file mode 100644 index 00000000000000..40489f41a2ad6a --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-03-26-12-27-42.gh-issue-146444.JKJuEa.rst @@ -0,0 +1 @@ +The Apple/iOS build script has been moved to the Platforms directory. diff --git a/Misc/NEWS.d/next/Build/2026-03-26-12-48-42.gh-issue-146446.0GyMu4.rst b/Misc/NEWS.d/next/Build/2026-03-26-12-48-42.gh-issue-146446.0GyMu4.rst new file mode 100644 index 00000000000000..40795650b53cbf --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-03-26-12-48-42.gh-issue-146446.0GyMu4.rst @@ -0,0 +1,2 @@ +The clean target for the Apple/iOS XCframework build script is now more +selective when targeting a single architecture. diff --git a/Misc/NEWS.d/next/Build/2026-03-26-14-35-29.gh-issue-146450.9Kmp5Q.rst b/Misc/NEWS.d/next/Build/2026-03-26-14-35-29.gh-issue-146450.9Kmp5Q.rst new file mode 100644 index 00000000000000..32cb5b8221a926 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-03-26-14-35-29.gh-issue-146450.9Kmp5Q.rst @@ -0,0 +1,2 @@ +The Android build script was modified to improve parity with other platform +build scripts. diff --git a/Misc/NEWS.d/next/Build/2026-03-27-06-55-10.gh-issue-146498.uOiCab.rst b/Misc/NEWS.d/next/Build/2026-03-27-06-55-10.gh-issue-146498.uOiCab.rst new file mode 100644 index 00000000000000..35deccd89761a3 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-03-27-06-55-10.gh-issue-146498.uOiCab.rst @@ -0,0 +1,3 @@ +The iOS XCframework build script now ensures libpython isn't included in +installed app content, and is more robust in identifying standard library +binary content that requires processing. diff --git a/Misc/NEWS.d/next/Build/2026-03-28-02-48-51.gh-issue-146541.k-zlM6.rst b/Misc/NEWS.d/next/Build/2026-03-28-02-48-51.gh-issue-146541.k-zlM6.rst new file mode 100644 index 00000000000000..351071b0becf66 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-03-28-02-48-51.gh-issue-146541.k-zlM6.rst @@ -0,0 +1 @@ +The Android testbed can now be built for 32-bit ARM and x86 targets. diff --git a/Misc/NEWS.d/next/C_API/2026-03-06-21-50-48.gh-issue-145559.AiXgHq.rst b/Misc/NEWS.d/next/C_API/2026-03-06-21-50-48.gh-issue-145559.AiXgHq.rst new file mode 100644 index 00000000000000..844e127ceae2a2 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-03-06-21-50-48.gh-issue-145559.AiXgHq.rst @@ -0,0 +1 @@ +Rename ``_Py_DumpTraceback`` and ``_Py_DumpTracebackThreads`` to :c:func:`PyUnstable_DumpTraceback` and :c:func:`PyUnstable_DumpTracebackThreads`. diff --git a/Misc/NEWS.d/next/C_API/2026-03-19-16-50-27.gh-issue-146175.pISQGX.rst b/Misc/NEWS.d/next/C_API/2026-03-19-16-50-27.gh-issue-146175.pISQGX.rst new file mode 100644 index 00000000000000..3563347141d1ba --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-03-19-16-50-27.gh-issue-146175.pISQGX.rst @@ -0,0 +1,12 @@ +The following macros are :term:`soft deprecated`: +:c:macro:`Py_ALIGNED`, +:c:macro:`PY_FORMAT_SIZE_T`, +:c:macro:`Py_LL`, :c:macro:`Py_ULL`, +:c:macro:`PY_LONG_LONG`, :c:macro:`PY_LLONG_MIN`, :c:macro:`PY_LLONG_MAX`, +:c:macro:`PY_ULLONG_MAX`, :c:macro:`PY_INT32_T`, :c:macro:`PY_UINT32_T`, +:c:macro:`PY_INT64_T`, :c:macro:`PY_UINT64_T`, :c:macro:`PY_SIZE_MAX`, +:c:macro:`Py_UNICODE_SIZE`, +:c:macro:`Py_VA_COPY`. + +The macro :c:macro:`Py_UNICODE_WIDE`, which was scheduled for removal, is +:term:`soft deprecated` instead. diff --git a/Misc/NEWS.d/next/C_API/2026-03-31-13-33-41.gh-issue-146636.5do3wt.rst b/Misc/NEWS.d/next/C_API/2026-03-31-13-33-41.gh-issue-146636.5do3wt.rst new file mode 100644 index 00000000000000..8f8b832b8baee9 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-03-31-13-33-41.gh-issue-146636.5do3wt.rst @@ -0,0 +1,3 @@ +The :c:data:`Py_mod_abi` slot is now mandatory for modules created from a +slots array (using :c:func:`PyModule_FromSlotsAndSpec` or the +:c:func:`PyModExport_* ` export hook). diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-01-01-49-52.gh-issue-140870.iknc12.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-01-01-49-52.gh-issue-140870.iknc12.rst new file mode 100644 index 00000000000000..aadf57622a424c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-01-01-49-52.gh-issue-140870.iknc12.rst @@ -0,0 +1,2 @@ +Add support for module attributes in the :term:`REPL` auto-completion of +imports. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-01-07-23-07-17.gh-issue-126910.d8zdm-.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-01-07-23-07-17.gh-issue-126910.d8zdm-.rst new file mode 100644 index 00000000000000..c86bfdb306f4c9 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-01-07-23-07-17.gh-issue-126910.d8zdm-.rst @@ -0,0 +1,2 @@ +Set frame pointers in ``x86_64-unknown-linux-gnu`` JIT code, allowing +most native profilers and debuggers to unwind through them. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-01-31-15-15-43.gh-issue-143414.Jgl4xu.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-01-31-15-15-43.gh-issue-143414.Jgl4xu.rst new file mode 100644 index 00000000000000..91f66e68bc9e8d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-01-31-15-15-43.gh-issue-143414.Jgl4xu.rst @@ -0,0 +1 @@ +Add tracking to the JIT optimizer to determine whether a reference is uniquely owned or shared diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-02-14-13-07-08.gh-issue-69605.4aL4hn.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-02-14-13-07-08.gh-issue-69605.4aL4hn.rst new file mode 100644 index 00000000000000..c00b7b99f8eea6 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-02-14-13-07-08.gh-issue-69605.4aL4hn.rst @@ -0,0 +1 @@ +Add :mod:`math.integer` to :term:`REPL` auto-completion of imports. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-13-12-24-17.gh-issue-145876.LWFO2K.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-13-12-24-17.gh-issue-145876.LWFO2K.rst new file mode 100644 index 00000000000000..86579634fa14ce --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-13-12-24-17.gh-issue-145876.LWFO2K.rst @@ -0,0 +1,3 @@ +:exc:`AttributeError`\ s and :exc:`KeyError`\ s raised in :meth:`!keys` or :meth:`!__getitem__` +during dictionary unpacking (``{**mymapping}`` or ``func(**mymapping)``) are +no longer masked by :exc:`TypeError`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-17-00-00-00.gh-issue-146041.7799bb.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-17-00-00-00.gh-issue-146041.7799bb.rst new file mode 100644 index 00000000000000..812f023266bd76 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-17-00-00-00.gh-issue-146041.7799bb.rst @@ -0,0 +1,3 @@ +Fix free-threading scaling bottleneck in :func:`sys.intern` and +:c:func:`PyObject_SetAttr` by avoiding the interpreter-wide lock when the string +is already interned and immortalized. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-17-14-20-56.gh-issue-145059.aB3xKm.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-17-14-20-56.gh-issue-145059.aB3xKm.rst new file mode 100644 index 00000000000000..e2db5a83a1a9e3 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-17-14-20-56.gh-issue-145059.aB3xKm.rst @@ -0,0 +1 @@ +Fixed ``sys.lazy_modules`` to include lazy modules without submodules. Patch by Bartosz SÅ‚awecki. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-18-18-52-00.gh-issue-146056.r1tVSo.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-18-18-52-00.gh-issue-146056.r1tVSo.rst index 67502657047dee..ab6eab2c968e8f 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-18-18-52-00.gh-issue-146056.r1tVSo.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-18-18-52-00.gh-issue-146056.r1tVSo.rst @@ -1 +1 @@ -Fix :meth:`!list.__repr__` for lists containing ``NULL``\ s. +Fix :func:`repr` for lists and tuples containing ``NULL``\ s. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-19-16-16-40.gh-issue-135871.jSExZ3.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-19-16-16-40.gh-issue-135871.jSExZ3.rst new file mode 100644 index 00000000000000..29103e46906487 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-19-16-16-40.gh-issue-135871.jSExZ3.rst @@ -0,0 +1 @@ +Improve multithreaded scaling of PyMutex in low-contention scenarios by reloading the lock's internal state, without slowing down high-contention scenarios. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-20-00-39-25.gh-issue-146192.8aQ6sC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-20-00-39-25.gh-issue-146192.8aQ6sC.rst new file mode 100644 index 00000000000000..304a7cd62102a7 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-20-00-39-25.gh-issue-146192.8aQ6sC.rst @@ -0,0 +1,2 @@ +Add Base32 support to :mod:`binascii` and improve the performance of the +Base32 converters in :mod:`base64`. Patch by James Seo. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-20-11-34-17.gh-issue-145667._Agp9o.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-20-11-34-17.gh-issue-145667._Agp9o.rst new file mode 100644 index 00000000000000..cedd8bfe6ce3bc --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-20-11-34-17.gh-issue-145667._Agp9o.rst @@ -0,0 +1,2 @@ +Remove the ``GET_ITER_YIELD_FROM`` instruction, modifying ``SEND`` to pair +with ``GET_ITER`` when compiling ``yield from`` expressions. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-20-12-26-24.gh-issue-146199.vV8V9s.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-20-12-26-24.gh-issue-146199.vV8V9s.rst new file mode 100644 index 00000000000000..0611a0d6a6d65a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-20-12-26-24.gh-issue-146199.vV8V9s.rst @@ -0,0 +1 @@ +Comparison of code objects now handles errors correctly. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-20-13-07-33.gh-issue-146227.MqBPEo.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-20-13-07-33.gh-issue-146227.MqBPEo.rst new file mode 100644 index 00000000000000..11e19eb28313d6 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-20-13-07-33.gh-issue-146227.MqBPEo.rst @@ -0,0 +1,3 @@ +Fix wrong type in ``_Py_atomic_load_uint16`` in the C11 atomics backend +(``pyatomic_std.h``), which used a 32-bit atomic load instead of 16-bit. +Found by Mohammed Zuhaib. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-21-08-11-58.gh-issue-146151.4-lhim.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-21-08-11-58.gh-issue-146151.4-lhim.rst new file mode 100644 index 00000000000000..d4a65d315110f4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-21-08-11-58.gh-issue-146151.4-lhim.rst @@ -0,0 +1,3 @@ +:class:`memoryview` now supports the :c:expr:`float complex` and +:c:expr:`double complex` C types: formatting characters ``'F'`` and ``'D'`` +respectively. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-21-08-48-25.gh-issue-146245.cqM3_4.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-21-08-48-25.gh-issue-146245.cqM3_4.rst new file mode 100644 index 00000000000000..f52eaa0d6c7277 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-21-08-48-25.gh-issue-146245.cqM3_4.rst @@ -0,0 +1 @@ +Fixed reference leaks in :mod:`socket` when audit hooks raise exceptions in :func:`socket.getaddrinfo` and :meth:`!socket.sendto`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-21-11-55-16.gh-issue-146250.ahl3O2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-21-11-55-16.gh-issue-146250.ahl3O2.rst new file mode 100644 index 00000000000000..fff07b31ec21c4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-21-11-55-16.gh-issue-146250.ahl3O2.rst @@ -0,0 +1 @@ +Fixed a memory leak in :exc:`SyntaxError` when re-initializing it. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-22-12-00-00.gh-issue-146306.870ef4.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-22-12-00-00.gh-issue-146306.870ef4.rst new file mode 100644 index 00000000000000..de2c3e56ac3319 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-22-12-00-00.gh-issue-146306.870ef4.rst @@ -0,0 +1,3 @@ +Optimize float arithmetic in the JIT by mutating uniquely-referenced +operands in place, avoiding allocation of a new float object. Speeds up +the pyperformance ``nbody`` benchmark by ~19%. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-22-19-30-00.gh-issue-146308.AxnRVA.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-22-19-30-00.gh-issue-146308.AxnRVA.rst new file mode 100644 index 00000000000000..9bc2f1c59a8c0c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-22-19-30-00.gh-issue-146308.AxnRVA.rst @@ -0,0 +1,5 @@ +Fixed multiple error handling issues in the :mod:`!_remote_debugging` module +including a double-free in code object caching, memory leaks on allocation +failure, missing exception checks in binary format varint decoding, reference +leaks on error paths in frame chain processing, and inconsistent thread status +error reporting across platforms. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-24-13-06-52.gh-issue-146369.6wDI6S.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-24-13-06-52.gh-issue-146369.6wDI6S.rst new file mode 100644 index 00000000000000..191b7627ed4e56 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-24-13-06-52.gh-issue-146369.6wDI6S.rst @@ -0,0 +1,2 @@ +Ensure ``-X lazy_imports=none``` and ``PYTHON_LAZY_IMPORTS=none``` override +``__lazy_modules__``. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-26-11-18-45.gh-issue-146388.O0u1c3.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-26-11-18-45.gh-issue-146388.O0u1c3.rst new file mode 100644 index 00000000000000..7cf5edfe8c6c6f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-26-11-18-45.gh-issue-146388.O0u1c3.rst @@ -0,0 +1 @@ +Adds a null check to handle when the JIT optimizer runs out of space when dealing with contradictions in ``make_bottom``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-27-17-14-18.gh-issue-126910.hooVFQ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-27-17-14-18.gh-issue-126910.hooVFQ.rst new file mode 100644 index 00000000000000..e3ddf39408804b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-27-17-14-18.gh-issue-126910.hooVFQ.rst @@ -0,0 +1 @@ +Set frame pointers in ``aarch64-unknown-linux-gnu`` JIT code, allowing most native profilers and debuggers to unwind through them. Patch by Diego Russo diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-29-11-39-05.gh-issue-146587.YJicXt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-29-11-39-05.gh-issue-146587.YJicXt.rst new file mode 100644 index 00000000000000..a33dee5c875389 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-29-11-39-05.gh-issue-146587.YJicXt.rst @@ -0,0 +1 @@ +Fix type slot assignment incase of multiple slots for same name in type object implementation. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-30-20-00-00.gh-issue-146306.C45609.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-30-20-00-00.gh-issue-146306.C45609.rst new file mode 100644 index 00000000000000..fdbdb6a285faff --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-30-20-00-00.gh-issue-146306.C45609.rst @@ -0,0 +1,3 @@ +Optimize compact integer arithmetic in the JIT by mutating +uniquely-referenced operands in place, avoiding allocation of a new int +object. Speeds up the pyperformance ``spectral_norm`` benchmark by ~10%. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-31-01-06-35.gh-issue-146615.fix-method-get.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-31-01-06-35.gh-issue-146615.fix-method-get.rst new file mode 100644 index 00000000000000..7a205f1d6dda61 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-31-01-06-35.gh-issue-146615.fix-method-get.rst @@ -0,0 +1,3 @@ +Fix a crash in :meth:`~object.__get__` for :c:expr:`METH_METHOD` descriptors +when an invalid (non-type) object is passed as the second argument. +Patch by Steven Sun. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-31-18-07-53.gh-issue-147856.62Dwee.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-31-18-07-53.gh-issue-147856.62Dwee.rst new file mode 100644 index 00000000000000..67ebd57b3a50f6 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-31-18-07-53.gh-issue-147856.62Dwee.rst @@ -0,0 +1 @@ +Allow the *count* argument of :meth:`bytes.replace` to be a keyword. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-01-12-35-55.gh-issue-147985.YVirHJ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-01-12-35-55.gh-issue-147985.YVirHJ.rst new file mode 100644 index 00000000000000..a94dfca5e2a498 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-01-12-35-55.gh-issue-147985.YVirHJ.rst @@ -0,0 +1,3 @@ +Make :c:func:`PySet_Contains` attempt a lock-free lookup, similar to +:meth:`!set.__contains__`. This avoids acquiring the set object mutex in the +normal case. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-01-12-52-31.gh-issue-144319.iZk4hs.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-01-12-52-31.gh-issue-144319.iZk4hs.rst new file mode 100644 index 00000000000000..f3f07ab35dbb01 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-01-12-52-31.gh-issue-144319.iZk4hs.rst @@ -0,0 +1,3 @@ +Fix a bug that could cause applications with specific allocation patterns to +leak memory via Huge Pages if compiled with Huge Page support. Patch by +Pablo Galindo diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-04-20-59-12.gh-issue-148083.9ZHNBN.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-04-20-59-12.gh-issue-148083.9ZHNBN.rst new file mode 100644 index 00000000000000..fea4659d0b9916 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-04-20-59-12.gh-issue-148083.9ZHNBN.rst @@ -0,0 +1 @@ +Constant-fold ``_CONTAINS_OP_SET`` for :class:`frozenset`. Patch by Donghee Na. diff --git a/Misc/NEWS.d/next/Documentation/2026-03-25-00-00-00.gh-issue-126676.052336.rst b/Misc/NEWS.d/next/Documentation/2026-03-25-00-00-00.gh-issue-126676.052336.rst new file mode 100644 index 00000000000000..d2e275fdf08385 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2026-03-25-00-00-00.gh-issue-126676.052336.rst @@ -0,0 +1,2 @@ +Expand :mod:`argparse` documentation for ``type=bool`` with a demonstration +of the surprising behavior and pointers to common alternatives. diff --git a/Misc/NEWS.d/next/Library/2019-04-25-21-11-37.bpo-36461.TO5YyP.rst b/Misc/NEWS.d/next/Library/2019-04-25-21-11-37.bpo-36461.TO5YyP.rst new file mode 100644 index 00000000000000..e78f66018077c2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-04-25-21-11-37.bpo-36461.TO5YyP.rst @@ -0,0 +1,3 @@ +Make the target time of :meth:`timeit.Timer.autorange` configurable +and add ``--target-time`` option to the command-line interface of +:mod:`timeit`. diff --git a/Misc/NEWS.d/next/Library/2025-09-19-13-54-54.gh-issue-130472.LODfdk.rst b/Misc/NEWS.d/next/Library/2025-09-19-13-54-54.gh-issue-130472.LODfdk.rst new file mode 100644 index 00000000000000..3d2a7f00d3e6a8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-19-13-54-54.gh-issue-130472.LODfdk.rst @@ -0,0 +1 @@ +Add fancycompleter and enable it by default when using pyrepl. This gives colored tab completion. diff --git a/Misc/NEWS.d/next/Library/2025-10-05-15-38-02.gh-issue-139633.l3P839.rst b/Misc/NEWS.d/next/Library/2025-10-05-15-38-02.gh-issue-139633.l3P839.rst new file mode 100644 index 00000000000000..94bd18074f8d2d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-05-15-38-02.gh-issue-139633.l3P839.rst @@ -0,0 +1,2 @@ +The :mod:`netrc` security check is now run once per parse rather than once +per entry. diff --git a/Misc/NEWS.d/next/Library/2025-10-13-16-43-36.gh-issue-140049.VvmAzN.rst b/Misc/NEWS.d/next/Library/2025-10-13-16-43-36.gh-issue-140049.VvmAzN.rst new file mode 100644 index 00000000000000..d9489fd0baab91 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-13-16-43-36.gh-issue-140049.VvmAzN.rst @@ -0,0 +1 @@ +:func:`traceback.format_exception_only` now colorizes exception notes. diff --git a/Misc/NEWS.d/next/Library/2025-11-15-23-14-30.gh-issue-138577.KbShrt.rst b/Misc/NEWS.d/next/Library/2025-11-15-23-14-30.gh-issue-138577.KbShrt.rst new file mode 100644 index 00000000000000..df24f62982a424 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-15-23-14-30.gh-issue-138577.KbShrt.rst @@ -0,0 +1,4 @@ +:func:`getpass.getpass` with non-empty ``echo_char`` now handles keyboard shortcuts +including Ctrl+A/E (cursor movement), Ctrl+K/U (kill line), Ctrl+W (erase word), +and Ctrl+V (literal next) by reading the terminal's control character settings +and processing them appropriately in non-canonical mode. Patch by Sanyam Khurana. diff --git a/Misc/NEWS.d/next/Library/2026-02-19-16-34-18.gh-issue-144270.wJRtSr.rst b/Misc/NEWS.d/next/Library/2026-02-19-16-34-18.gh-issue-144270.wJRtSr.rst new file mode 100644 index 00000000000000..b8a4374bc2d3ca --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-02-19-16-34-18.gh-issue-144270.wJRtSr.rst @@ -0,0 +1,3 @@ +Made the *tag* parameter of :class:`xml.etree.ElementTree.Element` and the +*parent* and *tag* parameters of :func:`xml.etree.ElementTree.SubElement` +positional-only, matching the behavior of the C accelerator. diff --git a/Misc/NEWS.d/next/Library/2026-02-26-20-13-16.gh-issue-145264.4pggX_.rst b/Misc/NEWS.d/next/Library/2026-02-26-20-13-16.gh-issue-145264.4pggX_.rst new file mode 100644 index 00000000000000..22d53fe8db1123 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-02-26-20-13-16.gh-issue-145264.4pggX_.rst @@ -0,0 +1,4 @@ +Base64 decoder (see :func:`binascii.a2b_base64`, :func:`base64.b64decode`, etc) no +longer ignores excess data after the first padded quad in non-strict +(default) mode. Instead, in conformance with :rfc:`4648`, section 3.3, it now ignores +the pad character, "=", if it is present before the end of the encoded data. diff --git a/Misc/NEWS.d/next/Library/2026-03-07-02-44-52.gh-issue-145616.x8Mf23.rst b/Misc/NEWS.d/next/Library/2026-03-07-02-44-52.gh-issue-145616.x8Mf23.rst new file mode 100644 index 00000000000000..131570a0e03daa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-07-02-44-52.gh-issue-145616.x8Mf23.rst @@ -0,0 +1 @@ +Detect Android sysconfig ABI correctly on 32-bit ARM Android on 64-bit ARM kernel diff --git a/Misc/NEWS.d/next/Library/2026-03-08-00-00-00.gh-issue-145650.LgRepr.rst b/Misc/NEWS.d/next/Library/2026-03-08-00-00-00.gh-issue-145650.LgRepr.rst new file mode 100644 index 00000000000000..243834d0bbd564 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-08-00-00-00.gh-issue-145650.LgRepr.rst @@ -0,0 +1,3 @@ +Add :meth:`~object.__repr__` support to :class:`logging.Formatter` and +:class:`logging.Filter`, showing the format string and filter name +respectively. diff --git a/Misc/NEWS.d/next/Library/2026-03-10-01-54-34.gh-issue-145719.okJRoK.rst b/Misc/NEWS.d/next/Library/2026-03-10-01-54-34.gh-issue-145719.okJRoK.rst new file mode 100644 index 00000000000000..b7e82a45675614 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-10-01-54-34.gh-issue-145719.okJRoK.rst @@ -0,0 +1 @@ +Add ``application/efi`` MIME type to :mod:`mimetypes`. diff --git a/Misc/NEWS.d/next/Library/2026-03-10-19-50-59.gh-issue-138122.CsoBEo.rst b/Misc/NEWS.d/next/Library/2026-03-10-19-50-59.gh-issue-138122.CsoBEo.rst new file mode 100644 index 00000000000000..2059557e8bb92e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-10-19-50-59.gh-issue-138122.CsoBEo.rst @@ -0,0 +1,4 @@ +The ``profiling.sampling`` module now supports differential flamegraph +visualization via ``--diff-flamegraph`` to compare two profiling runs. +Functions are colored red (regressions), blue (improvements), gray (neutral), +or purple (new). Elided stacks show code paths that disappeared between runs. diff --git a/Misc/NEWS.d/next/Library/2026-03-12-21-01-48.gh-issue-145883.lUvXcc.rst b/Misc/NEWS.d/next/Library/2026-03-12-21-01-48.gh-issue-145883.lUvXcc.rst new file mode 100644 index 00000000000000..2c17768c5189da --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-12-21-01-48.gh-issue-145883.lUvXcc.rst @@ -0,0 +1,2 @@ +:mod:`zoneinfo`: Fix heap buffer overflow reads from malformed TZif data. +Found by OSS Fuzz, issues :oss-fuzz:`492245058` and :oss-fuzz:`492230068`. diff --git a/Misc/NEWS.d/next/Library/2026-03-16-00-00-00.gh-issue-146004.xOptProp.rst b/Misc/NEWS.d/next/Library/2026-03-16-00-00-00.gh-issue-146004.xOptProp.rst new file mode 100644 index 00000000000000..234e6102c6a252 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-16-00-00-00.gh-issue-146004.xOptProp.rst @@ -0,0 +1,9 @@ +All :option:`-X` options from the Python command line are now propagated to +child processes spawned by :mod:`multiprocessing`, not just a hard-coded +subset. This makes the behavior consistent between default "spawn" and +"forkserver" start methods and the old "fork" start method. The options +that were previously not propagated are: ``context_aware_warnings``, +``cpu_count``, ``disable-remote-debug``, ``int_max_str_digits``, +``lazy_imports``, ``no_debug_ranges``, ``pathconfig_warnings``, ``perf``, +``perf_jit``, ``presite``, ``pycache_prefix``, ``thread_inherit_context``, +and ``warn_default_encoding``. diff --git a/Misc/NEWS.d/next/Library/2026-03-17-19-51-05.gh-issue-123471.oY4UR5.rst b/Misc/NEWS.d/next/Library/2026-03-17-19-51-05.gh-issue-123471.oY4UR5.rst new file mode 100644 index 00000000000000..8d2e1b970e8171 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-17-19-51-05.gh-issue-123471.oY4UR5.rst @@ -0,0 +1 @@ +Make concurrent iteration over :class:`itertools.zip_longest` safe under free-threading. diff --git a/Misc/NEWS.d/next/Library/2026-03-17-20-52-24.gh-issue-146083.NxZa_c.rst b/Misc/NEWS.d/next/Library/2026-03-17-20-52-24.gh-issue-146083.NxZa_c.rst new file mode 100644 index 00000000000000..6805a40a03e734 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-17-20-52-24.gh-issue-146083.NxZa_c.rst @@ -0,0 +1 @@ +Update bundled `libexpat `_ to version 2.7.5. diff --git a/Misc/NEWS.d/next/Library/2026-03-20-14-53-00.gh-issue-146228.OJVEDL.rst b/Misc/NEWS.d/next/Library/2026-03-20-14-53-00.gh-issue-146228.OJVEDL.rst new file mode 100644 index 00000000000000..1356e2ca07d9ae --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-20-14-53-00.gh-issue-146228.OJVEDL.rst @@ -0,0 +1,2 @@ +Cached FastPath objects in importlib.metadata are now cleared on fork, +avoiding broken references to zip files during fork. diff --git a/Misc/NEWS.d/next/Library/2026-03-20-16-17-31.gh-issue-143387.9Waopa.rst b/Misc/NEWS.d/next/Library/2026-03-20-16-17-31.gh-issue-143387.9Waopa.rst new file mode 100644 index 00000000000000..16bab047424e50 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-20-16-17-31.gh-issue-143387.9Waopa.rst @@ -0,0 +1,7 @@ +In importlib.metadata, when a distribution file is corrupt and there is no +metadata file, calls to ``Distribution.metadata()`` (including implicit +calls from other properties like ``.name`` and ``.requires``) will now raise +a ``MetadataNotFound`` Exception. This allows callers to distinguish between +missing metadata and a degenerate (empty) metadata. Previously, if the file +was missing, an empty ``PackageMetadata`` would be returned and would be +indistinguishable from the presence of an empty file. diff --git a/Misc/NEWS.d/next/Library/2026-03-21-06-21-38.gh-issue-146151.yNpgml.rst b/Misc/NEWS.d/next/Library/2026-03-21-06-21-38.gh-issue-146151.yNpgml.rst new file mode 100644 index 00000000000000..020b7d7d6cc3d0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-21-06-21-38.gh-issue-146151.yNpgml.rst @@ -0,0 +1,3 @@ +Support the :c:expr:`float complex` and :c:expr:`double complex` C types +in the :mod:`array` module: formatting characters ``'F'`` and ``'D'`` +respectively. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Library/2026-03-21-08-23-26.gh-issue-140947.owZ4r_.rst b/Misc/NEWS.d/next/Library/2026-03-21-08-23-26.gh-issue-140947.owZ4r_.rst new file mode 100644 index 00000000000000..88e787e8549a1d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-21-08-23-26.gh-issue-140947.owZ4r_.rst @@ -0,0 +1 @@ +Fix incorrect contextvars handling in server tasks created by :mod:`asyncio`. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2026-03-21-10-02-20.gh-issue-146238.2WpMOj.rst b/Misc/NEWS.d/next/Library/2026-03-21-10-02-20.gh-issue-146238.2WpMOj.rst new file mode 100644 index 00000000000000..35e951e38e4152 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-21-10-02-20.gh-issue-146238.2WpMOj.rst @@ -0,0 +1,2 @@ +Support half-floats (type code ``'e'`` of the :mod:`struct` module) in the +:mod:`array` module. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Library/2026-03-21-16-03-16.gh-issue-141510.tKptA7.rst b/Misc/NEWS.d/next/Library/2026-03-21-16-03-16.gh-issue-141510.tKptA7.rst new file mode 100644 index 00000000000000..19c30f11b33c70 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-21-16-03-16.gh-issue-141510.tKptA7.rst @@ -0,0 +1,2 @@ +Support :class:`frozendict` in :mod:`plistlib`, for serialization only. +Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2026-03-24-03-49-50.gh-issue-146310.WhlDir.rst b/Misc/NEWS.d/next/Library/2026-03-24-03-49-50.gh-issue-146310.WhlDir.rst new file mode 100644 index 00000000000000..b712595585201b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-24-03-49-50.gh-issue-146310.WhlDir.rst @@ -0,0 +1,2 @@ +The :mod:`ensurepip` module no longer looks for ``pip-*.whl`` wheel packages +in the current directory. diff --git a/Misc/NEWS.d/next/Library/2026-03-25-21-08-51.gh-issue-146431.zERPwe.rst b/Misc/NEWS.d/next/Library/2026-03-25-21-08-51.gh-issue-146431.zERPwe.rst new file mode 100644 index 00000000000000..6268a52926ffaa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-25-21-08-51.gh-issue-146431.zERPwe.rst @@ -0,0 +1,11 @@ +Add the *wrapcol* parameter to :mod:`base64` functions +:func:`~base64.b16encode`, :func:`~base64.b32encode`, +:func:`~base64.b32hexencode`, :func:`~base64.b85encode` and +:func:`~base64.z85encode`, and :mod:`binascii` functions +:func:`~binascii.b2a_base32` and :func:`~binascii.b2a_base85`. Add the +*ignorechars* parameter to :mod:`base64` functions +:func:`~base64.b16decode`, :func:`~base64.b32decode`, +:func:`~base64.b32hexdecode`, :func:`~base64.b85decode` and +:func:`~base64.z85decode`, and :mod:`binascii` functions +:func:`~binascii.a2b_hex`, :func:`~binascii.unhexlify`, +:func:`~binascii.a2b_base32` and :func:`~binascii.a2b_base85`. diff --git a/Misc/NEWS.d/next/Library/2026-03-26-02-06-52.gh-issue-146440.HXjhQO.rst b/Misc/NEWS.d/next/Library/2026-03-26-02-06-52.gh-issue-146440.HXjhQO.rst new file mode 100644 index 00000000000000..231c56fa063e72 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-26-02-06-52.gh-issue-146440.HXjhQO.rst @@ -0,0 +1,6 @@ +:mod:`json`: Add the *array_hook* parameter to :func:`~json.load` and +:func:`~json.loads` functions: +allow a callback for JSON literal array types to customize Python lists in the +resulting decoded object. Passing combined :class:`frozendict` to +*object_pairs_hook* param and :class:`tuple` to ``array_hook`` will yield a +deeply nested immutable Python structure representing the JSON data. diff --git a/Misc/NEWS.d/next/Library/2026-03-26-11-04-42.gh-issue-145633.RWjlaX.rst b/Misc/NEWS.d/next/Library/2026-03-26-11-04-42.gh-issue-145633.RWjlaX.rst new file mode 100644 index 00000000000000..00507fe89d07ec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-26-11-04-42.gh-issue-145633.RWjlaX.rst @@ -0,0 +1,2 @@ +Fix ``struct.pack('f', float)``: use :c:func:`PyFloat_Pack4` to raise +:exc:`OverflowError`. Patch by Sergey B Kirpichev and Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2026-03-26-14-44-07.gh-issue-145056.L9KPC3.rst b/Misc/NEWS.d/next/Library/2026-03-26-14-44-07.gh-issue-145056.L9KPC3.rst new file mode 100644 index 00000000000000..66e31117e33b6c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-26-14-44-07.gh-issue-145056.L9KPC3.rst @@ -0,0 +1 @@ +Add support for merging :class:`collections.UserDict` and :class:`frozendict`. diff --git a/Misc/NEWS.d/next/Library/2026-03-26-14-51-55.gh-issue-145056.QS-6l1.rst b/Misc/NEWS.d/next/Library/2026-03-26-14-51-55.gh-issue-145056.QS-6l1.rst new file mode 100644 index 00000000000000..4eaabfbb9a87a0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-26-14-51-55.gh-issue-145056.QS-6l1.rst @@ -0,0 +1 @@ +Fix merging of :class:`collections.OrderedDict` and :class:`frozendict`. diff --git a/Misc/NEWS.d/next/Library/2026-03-27-12-00-00.gh-issue-146507.1D95A7.rst b/Misc/NEWS.d/next/Library/2026-03-27-12-00-00.gh-issue-146507.1D95A7.rst new file mode 100644 index 00000000000000..f0aae2068fc9e6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-27-12-00-00.gh-issue-146507.1D95A7.rst @@ -0,0 +1,3 @@ +Make :meth:`asyncio.SelectorEventLoop` stream transport's +:meth:`~asyncio.WriteTransport.get_write_buffer_size` O(1) by maintaining a +running byte counter instead of iterating the buffer on every call. diff --git a/Misc/NEWS.d/next/Library/2026-03-28-12-01-48.gh-issue-146090.wh1qJR.rst b/Misc/NEWS.d/next/Library/2026-03-28-12-01-48.gh-issue-146090.wh1qJR.rst new file mode 100644 index 00000000000000..a6d60d2c929304 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-28-12-01-48.gh-issue-146090.wh1qJR.rst @@ -0,0 +1,2 @@ +:mod:`sqlite3`: properly raise :exc:`MemoryError` instead of :exc:`SystemError` +when a context callback fails to be allocated. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2026-03-28-12-05-34.gh-issue-146090.wf9_ef.rst b/Misc/NEWS.d/next/Library/2026-03-28-12-05-34.gh-issue-146090.wf9_ef.rst new file mode 100644 index 00000000000000..5b835b0271a604 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-28-12-05-34.gh-issue-146090.wf9_ef.rst @@ -0,0 +1,3 @@ +:mod:`sqlite3`: fix a crash when :meth:`sqlite3.Connection.create_collation` +fails with `SQLITE_BUSY `__. Patch by +Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2026-03-28-12-20-19.gh-issue-146556.Y8Eson.rst b/Misc/NEWS.d/next/Library/2026-03-28-12-20-19.gh-issue-146556.Y8Eson.rst new file mode 100644 index 00000000000000..71f84593edb522 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-28-12-20-19.gh-issue-146556.Y8Eson.rst @@ -0,0 +1,5 @@ +Fix :func:`annotationlib.get_annotations` hanging indefinitely when called +with ``eval_str=True`` on a callable that has a circular ``__wrapped__`` +chain (e.g. ``f.__wrapped__ = f``). Cycle detection using an id-based +visited set now stops the traversal and falls back to the globals found +so far, mirroring the approach of :func:`inspect.unwrap`. diff --git a/Misc/NEWS.d/next/Library/2026-03-28-13-19-20.gh-issue-146080.srN12a.rst b/Misc/NEWS.d/next/Library/2026-03-28-13-19-20.gh-issue-146080.srN12a.rst new file mode 100644 index 00000000000000..c80e8e05d480e5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-28-13-19-20.gh-issue-146080.srN12a.rst @@ -0,0 +1,2 @@ +:mod:`ssl`: fix a crash when an SNI callback tries to use an SSL object that +has already been garbage-collected. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2026-03-31-19-54-32.gh-issue-147944.3dn8GZ.rst b/Misc/NEWS.d/next/Library/2026-03-31-19-54-32.gh-issue-147944.3dn8GZ.rst new file mode 100644 index 00000000000000..7ba75bac79c0ee --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-31-19-54-32.gh-issue-147944.3dn8GZ.rst @@ -0,0 +1,4 @@ +Accepted range for the *bytes_per_sep* argument of :meth:`bytes.hex`, +:meth:`bytearray.hex`, :meth:`memoryview.hex`, and :func:`binascii.b2a_hex` +is now increased, so passing ``sys.maxsize`` and ``-sys.maxsize`` is now +valid. diff --git a/Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst b/Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst new file mode 100644 index 00000000000000..94e198e7b28ad8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst @@ -0,0 +1,2 @@ +:mod:`itertools`: Fix a crash in :func:`itertools.groupby` when +the grouper iterator is concurrently mutated. diff --git a/Misc/NEWS.d/next/Library/2026-04-01-18-17-55.gh-issue-73613.PLEebm.rst b/Misc/NEWS.d/next/Library/2026-04-01-18-17-55.gh-issue-73613.PLEebm.rst new file mode 100644 index 00000000000000..8c50972d3ca45a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-01-18-17-55.gh-issue-73613.PLEebm.rst @@ -0,0 +1,7 @@ +Add the *padded* parameter in functions related to Base32 and Base64 codecs +in the :mod:`binascii` and :mod:`base64` modules. +In the encoding functions it controls whether the pad character can be added +in the output, in the decoding functions it controls whether padding is +required in input. +Padding of input no longer required in :func:`base64.urlsafe_b64decode` +by default. diff --git a/Misc/NEWS.d/next/Security/2026-01-16-12-04-49.gh-issue-143930.zYC5x3.rst b/Misc/NEWS.d/next/Security/2026-01-16-12-04-49.gh-issue-143930.zYC5x3.rst index 0f27eae99a0dfd..c561023c3c2d7a 100644 --- a/Misc/NEWS.d/next/Security/2026-01-16-12-04-49.gh-issue-143930.zYC5x3.rst +++ b/Misc/NEWS.d/next/Security/2026-01-16-12-04-49.gh-issue-143930.zYC5x3.rst @@ -1 +1 @@ -Reject leading dashes in URLs passed to :func:`webbrowser.open` +Reject leading dashes in URLs passed to :func:`webbrowser.open`. diff --git a/Misc/NEWS.d/next/Tests/2026-03-24-00-15-58.gh-issue-146202.LgH6Bj.rst b/Misc/NEWS.d/next/Tests/2026-03-24-00-15-58.gh-issue-146202.LgH6Bj.rst new file mode 100644 index 00000000000000..ef869fe2617256 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-03-24-00-15-58.gh-issue-146202.LgH6Bj.rst @@ -0,0 +1,3 @@ +Fix a race condition in regrtest: make sure that the temporary directory is +created in the worker process. Previously, temp_cwd() could fail on Windows if +the "build" directory was not created. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-03-22-00-00-00.gh-issue-135953.IptOwg.rst b/Misc/NEWS.d/next/Tools-Demos/2026-03-22-00-00-00.gh-issue-135953.IptOwg.rst new file mode 100644 index 00000000000000..50f39a830de1b1 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2026-03-22-00-00-00.gh-issue-135953.IptOwg.rst @@ -0,0 +1,3 @@ +Properly identify the main thread in the Gecko profiler collector by +using a status flag from the interpreter state instead of relying on +:func:`threading.main_thread` in the collector process. diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index c79bbd2878271e..ed9c08016808bb 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -48,11 +48,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "9bd33bd279c0d7ea37b0f2d7e07c7c53b7053507" + "checksumValue": "9dfd09a3be37618cbcea380c2374b2b8f0288f57" }, { "algorithm": "SHA256", - "checksumValue": "d20997001462356b5ce3810ebf5256c8205f58462c64f21eb9bf80f8d1822b08" + "checksumValue": "26805a0d1a7a6a5cd8ead9cf7f4da29f63f0547a9ad41e80dba4ed9fe1943140" } ], "fileName": "Modules/expat/expat.h" @@ -62,11 +62,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "e658ee5d638ab326109282ff09f1541e27fff8c2" + "checksumValue": "da0328279276800cc747ea7da23886a3f402ccb3" }, { "algorithm": "SHA256", - "checksumValue": "dbe0582b8f8a8140aca97009e8760105ceed9e7df01ea9d8b3fe47cebf2e5b2d" + "checksumValue": "15a80e414e9e7c43edba64b1608a77c724387070138693f9e9bcca49c78a2df7" } ], "fileName": "Modules/expat/expat_external.h" @@ -174,11 +174,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "7d3d7d72aa56c53fb5b9e10c0e74e161381f0255" + "checksumValue": "0c74fbd48dd515c58eeb65b7e71b29da94be4694" }, { "algorithm": "SHA256", - "checksumValue": "f4f87aa0268d92f2b8f5e663788bfadd2e926477d0b061ed4463c02ad29a3e25" + "checksumValue": "861e7a50ce81f9f16b42d32a9caa4f817d962b274b2929b579511c6f76d348d4" } ], "fileName": "Modules/expat/xmlparse.c" @@ -188,11 +188,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "c8769fcb93f00272a6e6ca560be633649c817ff7" + "checksumValue": "7cff4d7210f046144f5fa635113f9c26f30fe3d3" }, { "algorithm": "SHA256", - "checksumValue": "5b81f0eb0e144b611dbd1bc9e6037075a16bff94f823d57a81eb2a3e4999e91a" + "checksumValue": "eaa6c327f9db4a5cec768d0c01927fea212d3ef4d4f970ebc0a98b9f3602784c" } ], "fileName": "Modules/expat/xmlrole.c" @@ -216,11 +216,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "63e4766a09e63760c6518670509198f8d638f4ad" + "checksumValue": "48b7aa6503302d4157c61a8763629f3236c23502" }, { "algorithm": "SHA256", - "checksumValue": "0ad3f915f2748dc91bf4e4b4a50cf40bf2c95769d0eca7e3b293a230d82bb779" + "checksumValue": "75da65603e99837fd3116f1453372efd556f9f97d8de73364594dd78b3c8ec54" } ], "fileName": "Modules/expat/xmltok.c" @@ -272,11 +272,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "41b8c8fc275882c76d4210b7d40a18e506b07147" + "checksumValue": "705842f8a09b09cc021d82a71ab03344bfd07b0a" }, { "algorithm": "SHA256", - "checksumValue": "b2188c7e5fa5b33e355cf6cf342dfb8f6e23859f2a6b1ddf79841d7f84f7b196" + "checksumValue": "f95a2b4b7efda40f5faf366537cb20a57dddbad9655859d2e304f5e75f6907cc" } ], "fileName": "Modules/expat/xmltok_ns.c" @@ -1730,14 +1730,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "461ecc8aa98ab1a68c2db788175665d1a4db640dc05bf0e289b6ea17122144ec" + "checksumValue": "9931f9860d18e6cf72d183eb8f309bfb96196c00e1d40caa978e95bc9aa978b6" } ], - "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_7_4/expat-2.7.4.tar.gz", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_7_5/expat-2.7.5.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.7.4:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.7.5:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1745,7 +1745,7 @@ "name": "expat", "originator": "Organization: Expat development team", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.7.4" + "versionInfo": "2.7.5" }, { "SPDXID": "SPDXRef-PACKAGE-hacl-star", diff --git a/Modules/_abc.c b/Modules/_abc.c index f87a5c702946bc..3c4e0280525e1e 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -976,6 +976,7 @@ _abcmodule_free(void *module) } static PyModuleDef_Slot _abcmodule_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _abcmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 8eb8e191530a33..fda7d1ef934da0 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -13,6 +13,7 @@ #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_runtime_init.h" // _Py_ID() +#include "pycore_tuple.h" // _PyTuple_FromPair #include // offsetof() @@ -829,14 +830,10 @@ future_add_done_callback(asyncio_state *state, FutureObj *fut, PyObject *arg, fut->fut_context0 = Py_NewRef(ctx); } else { - PyObject *tup = PyTuple_New(2); + PyObject *tup = _PyTuple_FromPair(arg, (PyObject *)ctx); if (tup == NULL) { return NULL; } - Py_INCREF(arg); - PyTuple_SET_ITEM(tup, 0, arg); - Py_INCREF(ctx); - PyTuple_SET_ITEM(tup, 1, (PyObject *)ctx); if (fut->fut_callbacks != NULL) { int err = PyList_Append(fut->fut_callbacks, tup); @@ -1503,14 +1500,12 @@ _asyncio_Future__callbacks_get_impl(FutureObj *self) Py_ssize_t i = 0; if (self->fut_callback0 != NULL) { - PyObject *tup0 = PyTuple_New(2); + assert(self->fut_context0 != NULL); + PyObject *tup0 = _PyTuple_FromPair(self->fut_callback0, self->fut_context0); if (tup0 == NULL) { Py_DECREF(callbacks); return NULL; } - PyTuple_SET_ITEM(tup0, 0, Py_NewRef(self->fut_callback0)); - assert(self->fut_context0 != NULL); - PyTuple_SET_ITEM(tup0, 1, Py_NewRef(self->fut_context0)); PyList_SET_ITEM(callbacks, i, tup0); i++; } @@ -2244,7 +2239,7 @@ enter_task(_PyThreadStateImpl *ts, PyObject *loop, PyObject *task) PyExc_RuntimeError, "Cannot enter into task %R while another " \ "task %R is being executed.", - task, ts->asyncio_running_task, NULL); + task, ts->asyncio_running_task); return -1; } @@ -2265,7 +2260,7 @@ leave_task(_PyThreadStateImpl *ts, PyObject *loop, PyObject *task) PyExc_RuntimeError, "Invalid attempt to leave task %R while " \ "task %R is entered.", - task, ts->asyncio_running_task ? ts->asyncio_running_task : Py_None, NULL); + task, ts->asyncio_running_task ? ts->asyncio_running_task : Py_None); return -1; } Py_CLEAR(ts->asyncio_running_task); @@ -2328,7 +2323,7 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, self->task_log_destroy_pending = 0; PyErr_Format(PyExc_TypeError, "a coroutine was expected, got %R", - coro, NULL); + coro); return -1; } @@ -4394,6 +4389,7 @@ module_exec(PyObject *mod) } static struct PyModuleDef_Slot module_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 3a1491e5b96f29..329aa8e117ec3c 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -452,6 +452,7 @@ bisect_modexec(PyObject *m) } static PyModuleDef_Slot bisect_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, bisect_modexec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index f3457a13c96c1f..7b8cbf3ed96184 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -783,6 +783,7 @@ _bz2_free(void *module) } static struct PyModuleDef_Slot _bz2_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _bz2_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index 2f2edbb05ab5c5..ff52bfd8291ac1 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -1113,6 +1113,7 @@ static PyMethodDef _codecs_functions[] = { }; static PyModuleDef_Slot _codecs_slots[] = { + _Py_ABI_SLOT, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 15c9aa41911822..4ff05727ebc8ce 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -2873,6 +2873,7 @@ collections_exec(PyObject *module) { #undef ADD_TYPE static struct PyModuleDef_Slot collections_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, collections_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_csv.c b/Modules/_csv.c index a3f840acbe8c0b..9d6190a11c4b10 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -1833,6 +1833,7 @@ csv_exec(PyObject *module) { } static PyModuleDef_Slot csv_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, csv_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 2c691c3766fc4d..57d3d78969f533 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -111,6 +111,7 @@ bytes(cdata) #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() #include "pycore_pyatomic_ft_wrappers.h" #include "pycore_object.h" +#include "pycore_tuple.h" // _PyTuple_FromPair #ifdef MS_WIN32 # include "pycore_modsupport.h" // _PyArg_NoKeywords() #endif @@ -3511,7 +3512,7 @@ _PyCData_set(ctypes_state *st, only it's object list. So we create a tuple, containing b_objects list PLUS the array itself, and return that! */ - return PyTuple_Pack(2, keep, value); + return _PyTuple_FromPair(keep, value); } PyErr_Format(PyExc_TypeError, "incompatible types, %s instance instead of %s instance", @@ -5332,8 +5333,7 @@ PyCArrayType_from_ctype(ctypes_state *st, PyObject *itemtype, Py_ssize_t length) len = PyLong_FromSsize_t(length); if (len == NULL) return NULL; - key = PyTuple_Pack(2, itemtype, len); - Py_DECREF(len); + key = _PyTuple_FromPairSteal(Py_NewRef(itemtype), len); if (!key) return NULL; @@ -6497,6 +6497,7 @@ module_free(void *module) } static PyModuleDef_Slot module_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _ctypes_mod_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 3b46fdf838b16f..83802605e1f4dc 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -845,6 +845,7 @@ _curses_panel_exec(PyObject *mod) } static PyModuleDef_Slot _curses_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _curses_panel_exec}, // XXX gh-103092: fix isolation. {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index fe9d6fe2763f36..000d7318557a6e 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -5631,6 +5631,7 @@ cursesmodule_exec(PyObject *module) /* Initialization function for the module */ static PyModuleDef_Slot cursesmodule_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, cursesmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 9d803dc94b64c7..163e499d957b2e 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -13,6 +13,7 @@ #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_object.h" // _PyObject_Init() #include "pycore_time.h" // _PyTime_ObjectToTime_t() +#include "pycore_tuple.h" // _PyTuple_FromPair #include "pycore_unicodeobject.h" // _PyUnicode_Copy() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_pyatomic_ft_wrappers.h" @@ -2692,7 +2693,7 @@ delta_divmod(PyObject *left, PyObject *right) Py_DECREF(divmod); return NULL; } - result = PyTuple_Pack(2, PyTuple_GET_ITEM(divmod, 0), delta); + result = _PyTuple_FromPair(PyTuple_GET_ITEM(divmod, 0), delta); Py_DECREF(delta); Py_DECREF(divmod); return result; @@ -4496,7 +4497,7 @@ timezone_getinitargs(PyObject *op, PyObject *Py_UNUSED(dummy)) PyDateTime_TimeZone *self = PyTimeZone_CAST(op); if (self->name == NULL) return PyTuple_Pack(1, self->offset); - return PyTuple_Pack(2, self->offset, self->name); + return _PyTuple_FromPair(self->offset, self->name); } static PyMethodDef timezone_methods[] = { @@ -5247,7 +5248,7 @@ time_getstate(PyDateTime_Time *self, int proto) if (! HASTZINFO(self) || self->tzinfo == Py_None) result = PyTuple_Pack(1, basestate); else - result = PyTuple_Pack(2, basestate, self->tzinfo); + result = _PyTuple_FromPair(basestate, self->tzinfo); Py_DECREF(basestate); } return result; @@ -7169,7 +7170,7 @@ datetime_getstate(PyDateTime_DateTime *self, int proto) if (! HASTZINFO(self) || self->tzinfo == Py_None) result = PyTuple_Pack(1, basestate); else - result = PyTuple_Pack(2, basestate, self->tzinfo); + result = _PyTuple_FromPair(basestate, self->tzinfo); Py_DECREF(basestate); } return result; @@ -7656,7 +7657,7 @@ _datetime_exec(PyObject *module) } static PyModuleDef_Slot module_slots[] = { - _Py_INTERNAL_ABI_SLOT, + _Py_ABI_SLOT, {Py_mod_exec, _datetime_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index f88861fa24423b..6b07ef74cfa51d 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -674,6 +674,7 @@ _dbm_module_free(void *module) } static PyModuleDef_Slot _dbmmodule_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _dbm_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index c42757e042e7ef..0a8308d9ebce7a 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -32,6 +32,7 @@ #include #include "pycore_object.h" // _PyObject_VisitType() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_tuple.h" // _PyTuple_FromPair #include "pycore_typeobject.h" #include @@ -3975,7 +3976,6 @@ _decimal_Decimal_as_integer_ratio_impl(PyObject *self, PyTypeObject *cls) PyObject *numerator = NULL; PyObject *denominator = NULL; PyObject *exponent = NULL; - PyObject *result = NULL; PyObject *tmp; mpd_ssize_t exp; PyObject *context; @@ -4035,6 +4035,7 @@ _decimal_Decimal_as_integer_ratio_impl(PyObject *self, PyTypeObject *cls) if (exp >= 0) { Py_SETREF(numerator, state->_py_long_multiply(numerator, exponent)); + Py_CLEAR(exponent); if (numerator == NULL) { goto error; } @@ -4061,15 +4062,13 @@ _decimal_Decimal_as_integer_ratio_impl(PyObject *self, PyTypeObject *cls) goto error; } } - - result = PyTuple_Pack(2, numerator, denominator); - + return _PyTuple_FromPairSteal(numerator, denominator); error: Py_XDECREF(exponent); Py_XDECREF(denominator); Py_XDECREF(numerator); - return result; + return NULL; } /*[clinic input] @@ -4613,7 +4612,6 @@ nm_mpd_qdivmod(PyObject *v, PyObject *w) PyObject *q, *r; PyObject *context; uint32_t status = 0; - PyObject *ret; decimal_state *state = find_state_left_or_right(v, w); CURRENT_CONTEXT(state, context); @@ -4642,10 +4640,7 @@ nm_mpd_qdivmod(PyObject *v, PyObject *w) return NULL; } - ret = PyTuple_Pack(2, q, r); - Py_DECREF(r); - Py_DECREF(q); - return ret; + return _PyTuple_FromPairSteal(q, r); } static PyObject * @@ -6674,7 +6669,6 @@ _decimal_Context_divmod_impl(PyObject *context, PyObject *x, PyObject *y) PyObject *a, *b; PyObject *q, *r; uint32_t status = 0; - PyObject *ret; CONVERT_BINOP_RAISE(&a, &b, x, y, context); decimal_state *state = get_module_state_from_ctx(context); @@ -6701,10 +6695,7 @@ _decimal_Context_divmod_impl(PyObject *context, PyObject *x, PyObject *y) return NULL; } - ret = PyTuple_Pack(2, q, r); - Py_DECREF(r); - Py_DECREF(q); - return ret; + return _PyTuple_FromPairSteal(q, r); } /* Binary or ternary arithmetic functions */ @@ -7810,15 +7801,15 @@ _decimal_exec(PyObject *m) switch (cm->flag) { case MPD_Float_operation: - base = PyTuple_Pack(2, state->DecimalException, PyExc_TypeError); + base = _PyTuple_FromPair(state->DecimalException, PyExc_TypeError); break; case MPD_Division_by_zero: - base = PyTuple_Pack(2, state->DecimalException, - PyExc_ZeroDivisionError); + base = _PyTuple_FromPair(state->DecimalException, + PyExc_ZeroDivisionError); break; case MPD_Overflow: - base = PyTuple_Pack(2, state->signal_map[INEXACT].ex, - state->signal_map[ROUNDED].ex); + base = _PyTuple_FromPair(state->signal_map[INEXACT].ex, + state->signal_map[ROUNDED].ex); break; case MPD_Underflow: base = PyTuple_Pack(3, state->signal_map[INEXACT].ex, @@ -7857,7 +7848,7 @@ _decimal_exec(PyObject *m) for (cm = state->cond_map+1; cm->name != NULL; cm++) { PyObject *base; if (cm->flag == MPD_Division_undefined) { - base = PyTuple_Pack(2, state->signal_map[0].ex, PyExc_ZeroDivisionError); + base = _PyTuple_FromPair(state->signal_map[0].ex, PyExc_ZeroDivisionError); } else { base = PyTuple_Pack(1, state->signal_map[0].ex); @@ -8029,6 +8020,7 @@ decimal_free(void *module) } static struct PyModuleDef_Slot _decimal_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _decimal_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index e0bc69c5fe22f8..e2185c4bd03aad 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -18,6 +18,7 @@ #include "Python.h" #include "pycore_dict.h" // _PyDict_CopyAsDict() #include "pycore_pyhash.h" // _Py_HashSecret +#include "pycore_tuple.h" // _PyTuple_FromPair #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include // offsetof() @@ -2594,7 +2595,7 @@ _elementtree__set_factories_impl(PyObject *module, PyObject *comment_factory, return NULL; } - old = PyTuple_Pack(2, + old = _PyTuple_FromPair( st->comment_factory ? st->comment_factory : Py_None, st->pi_factory ? st->pi_factory : Py_None); @@ -2712,7 +2713,7 @@ treebuilder_append_event(TreeBuilderObject *self, PyObject *action, { if (action != NULL) { PyObject *res; - PyObject *event = PyTuple_Pack(2, action, node); + PyObject *event = _PyTuple_FromPair(action, node); if (event == NULL) return -1; res = PyObject_CallOneArg(self->events_append, event); @@ -2840,8 +2841,6 @@ treebuilder_handle_data(TreeBuilderObject* self, PyObject* data) LOCAL(PyObject*) treebuilder_handle_end(TreeBuilderObject* self, PyObject* tag) { - PyObject* item; - if (treebuilder_flush_data(self) < 0) { return NULL; } @@ -2854,17 +2853,22 @@ treebuilder_handle_end(TreeBuilderObject* self, PyObject* tag) return NULL; } - item = self->last; - self->last = Py_NewRef(self->this); - Py_XSETREF(self->last_for_tail, self->last); + PyObject *last = self->last; + PyObject *last_for_tail = self->last_for_tail; + PyObject *this = self->this; + self->last = Py_NewRef(this); + self->last_for_tail = Py_NewRef(this); self->index--; self->this = Py_NewRef(PyList_GET_ITEM(self->stack, self->index)); - Py_DECREF(item); + Py_DECREF(last); + Py_XDECREF(last_for_tail); - if (treebuilder_append_event(self, self->end_event_obj, self->last) < 0) + if (treebuilder_append_event(self, self->end_event_obj, self->last) < 0) { + Py_DECREF(this); return NULL; + } - return Py_NewRef(self->last); + return this; } LOCAL(PyObject*) @@ -2930,7 +2934,7 @@ treebuilder_handle_pi(TreeBuilderObject* self, PyObject* target, PyObject* text) Py_XSETREF(self->last_for_tail, Py_NewRef(pi)); } } else { - pi = PyTuple_Pack(2, target, text); + pi = _PyTuple_FromPair(target, text); if (!pi) { return NULL; } @@ -2954,7 +2958,7 @@ treebuilder_handle_start_ns(TreeBuilderObject* self, PyObject* prefix, PyObject* PyObject* parcel; if (self->events_append && self->start_ns_event_obj) { - parcel = PyTuple_Pack(2, prefix, uri); + parcel = _PyTuple_FromPair(prefix, uri); if (!parcel) { return NULL; } @@ -4530,6 +4534,7 @@ module_exec(PyObject *m) } static struct PyModuleDef_Slot elementtree_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 576494e846ca0c..19bdf3d47c2fad 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -2018,6 +2018,7 @@ _functools_free(void *module) } static struct PyModuleDef_Slot _functools_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _functools_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index 72f568ceb06987..faffe8d28c5b5e 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -912,6 +912,7 @@ _gdbm_module_free(void *module) } static PyModuleDef_Slot _gdbm_module_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _gdbm_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index e19eb1abcf2c4d..938a6ce5b962d1 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -2899,6 +2899,7 @@ hashlib_constants(PyObject *module) } static PyModuleDef_Slot hashlib_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, hashlib_init_hashtable}, {Py_mod_exec, hashlib_init_HASH_type}, {Py_mod_exec, hashlib_init_HASHXOF_type}, diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 05d01acd77109b..c705376f4edbf0 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -786,6 +786,7 @@ heapq_exec(PyObject *m) } static struct PyModuleDef_Slot heapq_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, heapq_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index 2933332ad465d4..3c356cb40d2bca 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -3605,6 +3605,7 @@ module_exec(PyObject *mod) } static struct PyModuleDef_Slot module_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index 417c5fbcee2645..777b6854749884 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -1898,6 +1898,7 @@ module_exec(PyObject *mod) } static struct PyModuleDef_Slot module_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 2aee8b07891c91..4c9be1d525d587 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -15,6 +15,7 @@ #include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() #include "pycore_pylifecycle.h" // _PyInterpreterConfig_AsDict() #include "pycore_pystate.h" // _PyInterpreterState_IsRunningMain() +#include "pycore_tuple.h" // _PyTuple_FromPairSteal #include "marshal.h" // PyMarshal_ReadObjectFromString() @@ -796,10 +797,7 @@ get_summary(PyInterpreterState *interp) Py_DECREF(idobj); return NULL; } - PyObject *res = PyTuple_Pack(2, idobj, whenceobj); - Py_DECREF(idobj); - Py_DECREF(whenceobj); - return res; + return _PyTuple_FromPairSteal(idobj, whenceobj); } @@ -1634,6 +1632,7 @@ module_exec(PyObject *mod) } static struct PyModuleDef_Slot module_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 433d68d515ccc6..32c55f8e225ed9 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -722,6 +722,7 @@ iomodule_exec(PyObject *m) } static struct PyModuleDef_Slot iomodule_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, iomodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 781ca4327f93ae..5debae5b42480b 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -111,7 +111,7 @@ resize_buffer(stringio *self, size_t size) alloc = size + 1; } - if (alloc > PY_SIZE_MAX / sizeof(Py_UCS4)) + if (alloc > SIZE_MAX / sizeof(Py_UCS4)) goto overflow; new_buf = (Py_UCS4 *)PyMem_Realloc(self->buf, alloc * sizeof(Py_UCS4)); if (new_buf == NULL) { diff --git a/Modules/_json.c b/Modules/_json.c index cbede8f44dc065..36614138501e79 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -14,6 +14,7 @@ #include "pycore_global_strings.h" // _Py_ID() #include "pycore_pyerrors.h" // _PyErr_FormatNote #include "pycore_runtime.h" // _PyRuntime +#include "pycore_tuple.h" // _PyTuple_FromPair #include "pycore_unicodeobject.h" // _PyUnicode_CheckConsistency() #include // bool @@ -30,6 +31,7 @@ typedef struct _PyScannerObject { signed char strict; PyObject *object_hook; PyObject *object_pairs_hook; + PyObject *array_hook; PyObject *parse_float; PyObject *parse_int; PyObject *parse_constant; @@ -41,6 +43,7 @@ static PyMemberDef scanner_members[] = { {"strict", Py_T_BOOL, offsetof(PyScannerObject, strict), Py_READONLY, "strict"}, {"object_hook", _Py_T_OBJECT, offsetof(PyScannerObject, object_hook), Py_READONLY, "object_hook"}, {"object_pairs_hook", _Py_T_OBJECT, offsetof(PyScannerObject, object_pairs_hook), Py_READONLY}, + {"array_hook", _Py_T_OBJECT, offsetof(PyScannerObject, array_hook), Py_READONLY}, {"parse_float", _Py_T_OBJECT, offsetof(PyScannerObject, parse_float), Py_READONLY, "parse_float"}, {"parse_int", _Py_T_OBJECT, offsetof(PyScannerObject, parse_int), Py_READONLY, "parse_int"}, {"parse_constant", _Py_T_OBJECT, offsetof(PyScannerObject, parse_constant), Py_READONLY, "parse_constant"}, @@ -444,7 +447,6 @@ raise_stop_iteration(Py_ssize_t idx) static PyObject * _build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) { /* return (rval, idx) tuple, stealing reference to rval */ - PyObject *tpl; PyObject *pyidx; /* steal a reference to rval, returns (rval, idx) @@ -457,15 +459,7 @@ _build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) { Py_DECREF(rval); return NULL; } - tpl = PyTuple_New(2); - if (tpl == NULL) { - Py_DECREF(pyidx); - Py_DECREF(rval); - return NULL; - } - PyTuple_SET_ITEM(tpl, 0, rval); - PyTuple_SET_ITEM(tpl, 1, pyidx); - return tpl; + return _PyTuple_FromPairSteal(rval, pyidx); } static PyObject * @@ -720,6 +714,7 @@ scanner_traverse(PyObject *op, visitproc visit, void *arg) Py_VISIT(Py_TYPE(self)); Py_VISIT(self->object_hook); Py_VISIT(self->object_pairs_hook); + Py_VISIT(self->array_hook); Py_VISIT(self->parse_float); Py_VISIT(self->parse_int); Py_VISIT(self->parse_constant); @@ -732,6 +727,7 @@ scanner_clear(PyObject *op) PyScannerObject *self = PyScannerObject_CAST(op); Py_CLEAR(self->object_hook); Py_CLEAR(self->object_pairs_hook); + Py_CLEAR(self->array_hook); Py_CLEAR(self->parse_float); Py_CLEAR(self->parse_int); Py_CLEAR(self->parse_constant); @@ -806,11 +802,10 @@ _parse_object_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ss goto bail; if (has_pairs_hook) { - PyObject *item = PyTuple_Pack(2, key, val); + PyObject *item = _PyTuple_FromPairSteal(key, val); + key = val = NULL; if (item == NULL) goto bail; - Py_CLEAR(key); - Py_CLEAR(val); if (PyList_Append(rval, item) == -1) { Py_DECREF(item); goto bail; @@ -942,6 +937,12 @@ _parse_array_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ssi goto bail; } *next_idx_ptr = idx + 1; + /* if array_hook is not None: return array_hook(rval) */ + if (!Py_IsNone(s->array_hook)) { + val = PyObject_CallOneArg(s->array_hook, rval); + Py_DECREF(rval); + return val; + } return rval; bail: Py_XDECREF(val); @@ -1259,6 +1260,10 @@ scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds) s->object_pairs_hook = PyObject_GetAttrString(ctx, "object_pairs_hook"); if (s->object_pairs_hook == NULL) goto bail; + s->array_hook = PyObject_GetAttrString(ctx, "array_hook"); + if (s->array_hook == NULL) { + goto bail; + } s->parse_float = PyObject_GetAttrString(ctx, "parse_float"); if (s->parse_float == NULL) goto bail; @@ -2087,6 +2092,7 @@ _json_exec(PyObject *module) } static PyModuleDef_Slot _json_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _json_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 86a390e52a554b..8f7d662b00b21b 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -1052,6 +1052,7 @@ _locale_exec(PyObject *module) } static struct PyModuleDef_Slot _locale_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _locale_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index a2d1aefb1611b3..abb8db1acabbd5 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -1125,6 +1125,7 @@ _lsprof_exec(PyObject *module) } static PyModuleDef_Slot _lsprofslots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _lsprof_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index cd0d09682fac69..3c391675d7b93e 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -1594,6 +1594,7 @@ static PyMethodDef lzma_methods[] = { }; static PyModuleDef_Slot lzma_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, lzma_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 848784dedc1702..201cedbb59818f 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -274,6 +274,7 @@ multiprocessing_exec(PyObject *module) } static PyModuleDef_Slot multiprocessing_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, multiprocessing_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 2fe09593a457e9..dedf17f76dfc9b 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -427,6 +427,7 @@ _opcode_exec(PyObject *m) { } static PyModuleDef_Slot module_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _opcode_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_operator.c b/Modules/_operator.c index 1cc05c39f5dbad..417403dc4c10c1 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1194,7 +1194,7 @@ itemgetter_reduce(PyObject *op, PyObject *Py_UNUSED(dummy)) itemgetterobject *ig = itemgetterobject_CAST(op); if (ig->nitems == 1) return Py_BuildValue("O(O)", Py_TYPE(ig), ig->item); - return PyTuple_Pack(2, Py_TYPE(ig), ig->item); + return _PyTuple_FromPair((PyObject *)Py_TYPE(ig), ig->item); } PyDoc_STRVAR(reduce_doc, "Return state information for pickling"); @@ -1981,6 +1981,7 @@ operator_exec(PyObject *module) static struct PyModuleDef_Slot operator_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, operator_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 65facaa6db2036..9874f9475ac029 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -21,6 +21,7 @@ #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_symtable.h" // _Py_Mangle() #include "pycore_sysmodule.h" // _PySys_GetSizeOf() +#include "pycore_tuple.h" // _PyTuple_FromPair #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() #include // strtol() @@ -3671,16 +3672,13 @@ save_set(PickleState *state, PicklerObject *self, PyObject *obj) } static int -save_frozenset(PickleState *state, PicklerObject *self, PyObject *obj) +save_frozenset_impl(PickleState *state, PicklerObject *self, PyObject *obj) { PyObject *iter; const char mark_op = MARK; const char frozenset_op = FROZENSET; - if (self->fast && !fast_save_enter(self, obj)) - return -1; - if (self->proto < 4) { PyObject *items; PyObject *reduce_value; @@ -3751,13 +3749,26 @@ save_frozenset(PickleState *state, PicklerObject *self, PyObject *obj) return 0; } +static int +save_frozenset(PickleState *state, PicklerObject *self, PyObject *obj) +{ + if (self->fast && !fast_save_enter(self, obj)) { + return -1; + } + int status = save_frozenset_impl(state, self, obj); + if (self->fast && !fast_save_leave(self, obj)) { + return -1; + } + return status; +} + static int fix_imports(PickleState *st, PyObject **module_name, PyObject **global_name) { PyObject *key; PyObject *item; - key = PyTuple_Pack(2, *module_name, *global_name); + key = _PyTuple_FromPair(*module_name, *global_name); if (key == NULL) return -1; item = PyDict_GetItemWithError(st->name_mapping_3to2, key); @@ -3863,7 +3874,7 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, char pdata[5]; Py_ssize_t n; - extension_key = PyTuple_Pack(2, module_name, global_name); + extension_key = _PyTuple_FromPair(module_name, global_name); if (extension_key == NULL) { goto error; } @@ -5123,26 +5134,19 @@ static PyObject * _pickle_PicklerMemoProxy___reduce___impl(PicklerMemoProxyObject *self) /*[clinic end generated code: output=bebba1168863ab1d input=2f7c540e24b7aae4]*/ { - PyObject *reduce_value, *dict_args; + PyObject *dict_args; PyObject *contents = _pickle_PicklerMemoProxy_copy_impl(self); if (contents == NULL) return NULL; - reduce_value = PyTuple_New(2); - if (reduce_value == NULL) { - Py_DECREF(contents); - return NULL; - } dict_args = PyTuple_New(1); if (dict_args == NULL) { Py_DECREF(contents); - Py_DECREF(reduce_value); return NULL; } PyTuple_SET_ITEM(dict_args, 0, contents); - PyTuple_SET_ITEM(reduce_value, 0, Py_NewRef(&PyDict_Type)); - PyTuple_SET_ITEM(reduce_value, 1, dict_args); - return reduce_value; + + return _PyTuple_FromPairSteal(Py_NewRef(&PyDict_Type), dict_args); } static PyMethodDef picklerproxy_methods[] = { @@ -7300,7 +7304,7 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self, PyTypeObject *cls, /* Check if the global (i.e., a function or a class) was renamed or moved to another module. */ - key = PyTuple_Pack(2, module_name, global_name); + key = _PyTuple_FromPair(module_name, global_name); if (key == NULL) return NULL; item = PyDict_GetItemWithError(st->name_mapping_2to3, key); @@ -7630,27 +7634,19 @@ static PyObject * _pickle_UnpicklerMemoProxy___reduce___impl(UnpicklerMemoProxyObject *self) /*[clinic end generated code: output=6da34ac048d94cca input=6920862413407199]*/ { - PyObject *reduce_value; PyObject *constructor_args; PyObject *contents = _pickle_UnpicklerMemoProxy_copy_impl(self); if (contents == NULL) return NULL; - reduce_value = PyTuple_New(2); - if (reduce_value == NULL) { - Py_DECREF(contents); - return NULL; - } constructor_args = PyTuple_New(1); if (constructor_args == NULL) { Py_DECREF(contents); - Py_DECREF(reduce_value); return NULL; } PyTuple_SET_ITEM(constructor_args, 0, contents); - PyTuple_SET_ITEM(reduce_value, 0, Py_NewRef(&PyDict_Type)); - PyTuple_SET_ITEM(reduce_value, 1, constructor_args); - return reduce_value; + + return _PyTuple_FromPairSteal(Py_NewRef(&PyDict_Type), constructor_args); } static PyMethodDef unpicklerproxy_methods[] = { @@ -8240,6 +8236,7 @@ _pickle_exec(PyObject *m) } static PyModuleDef_Slot pickle_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _pickle_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 6f0a6d1d4e37fe..b7f39ea3d499e4 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -1337,6 +1337,7 @@ static PyMethodDef module_methods[] = { }; static PyModuleDef_Slot _posixsubprocess_slots[] = { + _Py_ABI_SLOT, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index f2246dd36cf110..ed925f3525a9a7 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -617,6 +617,7 @@ queuemodule_exec(PyObject *module) } static PyModuleDef_Slot queuemodule_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, queuemodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 544e636d18fede..0fb73481651748 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -643,6 +643,7 @@ _random_exec(PyObject *module) } static PyModuleDef_Slot _random_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _random_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index 7bcb2f483234ec..3722273dfd2998 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -172,6 +172,7 @@ typedef enum _WIN32_THREADSTATE { #define THREAD_STATUS_UNKNOWN (1 << 2) #define THREAD_STATUS_GIL_REQUESTED (1 << 3) #define THREAD_STATUS_HAS_EXCEPTION (1 << 4) +#define THREAD_STATUS_MAIN_THREAD (1 << 5) /* Exception cause macro */ #define set_exception_cause(unwinder, exc_type, message) \ @@ -307,6 +308,7 @@ typedef struct { #endif #ifdef __APPLE__ uint64_t thread_id_offset; + int thread_id_offset_initialized; #endif #ifdef MS_WINDOWS PVOID win_process_buffer; @@ -575,7 +577,8 @@ extern PyObject* unwind_stack_for_thread( RemoteUnwinderObject *unwinder, uintptr_t *current_tstate, uintptr_t gil_holder_tstate, - uintptr_t gc_frame + uintptr_t gc_frame, + uintptr_t main_thread_tstate ); /* Thread stopping functions (for blocking mode) */ diff --git a/Modules/_remote_debugging/asyncio.c b/Modules/_remote_debugging/asyncio.c index 69478634de6926..263c502a857004 100644 --- a/Modules/_remote_debugging/asyncio.c +++ b/Modules/_remote_debugging/asyncio.c @@ -212,7 +212,7 @@ parse_task_name( set_exception_cause(unwinder, PyExc_RuntimeError, "Task name PyLong parsing failed"); return NULL; } - return PyUnicode_FromFormat("Task-%d", res); + return PyUnicode_FromFormat("Task-%ld", res); } if(!(GET_MEMBER(unsigned long, type_obj, unwinder->debug_offsets.type_object.tp_flags) & Py_TPFLAGS_UNICODE_SUBCLASS)) { @@ -265,7 +265,7 @@ handle_yield_from_frame( uintptr_t gi_await_addr; err = read_py_ptr( unwinder, - stackpointer_addr - sizeof(void*), + stackpointer_addr - sizeof(void*) * 2, &gi_await_addr); if (err) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read gi_await address"); @@ -940,6 +940,9 @@ process_running_task_chain( PyObject *coro_chain = PyStructSequence_GET_ITEM(task_info, 2); assert(coro_chain != NULL); if (PyList_GET_SIZE(coro_chain) != 1) { + PyErr_Format(PyExc_RuntimeError, + "Expected single-item coro chain, got %zd items", + PyList_GET_SIZE(coro_chain)); set_exception_cause(unwinder, PyExc_RuntimeError, "Coro chain is not a single item"); return -1; } diff --git a/Modules/_remote_debugging/binary_io.h b/Modules/_remote_debugging/binary_io.h index f8399f4aebe74b..d90546078bf68c 100644 --- a/Modules/_remote_debugging/binary_io.h +++ b/Modules/_remote_debugging/binary_io.h @@ -415,8 +415,8 @@ decode_varint_u32(const uint8_t *data, size_t *offset, size_t max_size) { size_t saved_offset = *offset; uint64_t value = decode_varint_u64(data, offset, max_size); - if (PyErr_Occurred()) { - return 0; + if (*offset == saved_offset) { + return 0; /* decode_varint_u64 already set PyErr */ } if (UNLIKELY(value > UINT32_MAX)) { *offset = saved_offset; @@ -430,9 +430,10 @@ decode_varint_u32(const uint8_t *data, size_t *offset, size_t max_size) static inline int32_t decode_varint_i32(const uint8_t *data, size_t *offset, size_t max_size) { + size_t saved_offset = *offset; uint32_t zigzag = decode_varint_u32(data, offset, max_size); - if (PyErr_Occurred()) { - return 0; + if (*offset == saved_offset) { + return 0; /* decode_varint_u32 already set PyErr */ } return (int32_t)((zigzag >> 1) ^ -(int32_t)(zigzag & 1)); } diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index cb58a0ed199d4a..616213541e12e1 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -571,15 +571,16 @@ reader_get_or_create_thread_state(BinaryReader *reader, uint64_t thread_id, return NULL; } } else if (reader->thread_state_count >= reader->thread_state_capacity) { - reader->thread_states = grow_array(reader->thread_states, - &reader->thread_state_capacity, - sizeof(ReaderThreadState)); - if (!reader->thread_states) { + ReaderThreadState *new_states = grow_array(reader->thread_states, + &reader->thread_state_capacity, + sizeof(ReaderThreadState)); + if (!new_states) { return NULL; } + reader->thread_states = new_states; } - ReaderThreadState *ts = &reader->thread_states[reader->thread_state_count++]; + ReaderThreadState *ts = &reader->thread_states[reader->thread_state_count]; memset(ts, 0, sizeof(ReaderThreadState)); ts->thread_id = thread_id; ts->interpreter_id = interpreter_id; @@ -590,6 +591,9 @@ reader_get_or_create_thread_state(BinaryReader *reader, uint64_t thread_id, PyErr_NoMemory(); return NULL; } + // Increment count only after successful allocation to avoid + // leaving a half-initialized entry visible to future lookups + reader->thread_state_count++; return ts; } @@ -604,7 +608,11 @@ static inline int decode_stack_full(ReaderThreadState *ts, const uint8_t *data, size_t *offset, size_t max_size) { + size_t prev_offset = *offset; uint32_t depth = decode_varint_u32(data, offset, max_size); + if (*offset == prev_offset) { + return -1; + } /* Validate depth against capacity to prevent buffer overflow */ if (depth > ts->current_stack_capacity) { @@ -615,7 +623,11 @@ decode_stack_full(ReaderThreadState *ts, const uint8_t *data, ts->current_stack_depth = depth; for (uint32_t i = 0; i < depth; i++) { + size_t prev_offset = *offset; ts->current_stack[i] = decode_varint_u32(data, offset, max_size); + if (*offset == prev_offset) { + return -1; + } } return 0; } @@ -627,8 +639,16 @@ static inline int decode_stack_suffix(ReaderThreadState *ts, const uint8_t *data, size_t *offset, size_t max_size) { + size_t prev_offset = *offset; uint32_t shared = decode_varint_u32(data, offset, max_size); + if (*offset == prev_offset) { + return -1; + } + prev_offset = *offset; uint32_t new_count = decode_varint_u32(data, offset, max_size); + if (*offset == prev_offset) { + return -1; + } /* Validate shared doesn't exceed current stack depth */ if (shared > ts->current_stack_depth) { @@ -664,7 +684,11 @@ decode_stack_suffix(ReaderThreadState *ts, const uint8_t *data, } for (uint32_t i = 0; i < new_count; i++) { + size_t prev_offset = *offset; ts->current_stack[i] = decode_varint_u32(data, offset, max_size); + if (*offset == prev_offset) { + return -1; + } } ts->current_stack_depth = final_depth; return 0; @@ -677,8 +701,16 @@ static inline int decode_stack_pop_push(ReaderThreadState *ts, const uint8_t *data, size_t *offset, size_t max_size) { + size_t prev_offset = *offset; uint32_t pop = decode_varint_u32(data, offset, max_size); + if (*offset == prev_offset) { + return -1; + } + prev_offset = *offset; uint32_t push = decode_varint_u32(data, offset, max_size); + if (*offset == prev_offset) { + return -1; + } size_t keep = (ts->current_stack_depth > pop) ? ts->current_stack_depth - pop : 0; /* Validate final depth doesn't exceed capacity */ @@ -699,7 +731,12 @@ decode_stack_pop_push(ReaderThreadState *ts, const uint8_t *data, } for (uint32_t i = 0; i < push; i++) { + size_t prev_offset = *offset; ts->current_stack[i] = decode_varint_u32(data, offset, max_size); + /* If offset didn't advance, varint decoding failed */ + if (*offset == prev_offset) { + return -1; + } } ts->current_stack_depth = final_depth; return 0; @@ -1222,6 +1259,9 @@ binary_reader_close(BinaryReader *reader) reader->mapped_data = NULL; /* Prevent use-after-free */ reader->mapped_size = 0; } + /* Clear sample_data which may point into the now-unmapped region */ + reader->sample_data = NULL; + reader->sample_data_size = 0; if (reader->fd >= 0) { close(reader->fd); reader->fd = -1; /* Mark as closed */ diff --git a/Modules/_remote_debugging/code_objects.c b/Modules/_remote_debugging/code_objects.c index 91f7a02005391a..7b95c0f2d4fa8d 100644 --- a/Modules/_remote_debugging/code_objects.c +++ b/Modules/_remote_debugging/code_objects.c @@ -110,6 +110,7 @@ cache_tlbc_array(RemoteUnwinderObject *unwinder, uintptr_t code_addr, uintptr_t void *key = (void *)code_addr; if (_Py_hashtable_set(unwinder->tlbc_cache, key, entry) < 0) { tlbc_cache_entry_destroy(entry); + PyErr_NoMemory(); set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to store TLBC entry in cache"); return 0; // Cache error } @@ -408,7 +409,14 @@ parse_code_object(RemoteUnwinderObject *unwinder, meta->addr_code_adaptive = real_address + (uintptr_t)unwinder->debug_offsets.code_object.co_code_adaptive; if (unwinder && unwinder->code_object_cache && _Py_hashtable_set(unwinder->code_object_cache, key, meta) < 0) { + // Ownership of func/file/linetable was transferred to meta, + // so NULL them before destroying meta to prevent double-free + // in the error label's Py_XDECREF calls. + func = NULL; + file = NULL; + linetable = NULL; cached_code_metadata_destroy(meta); + PyErr_NoMemory(); set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to cache code metadata"); goto error; } diff --git a/Modules/_remote_debugging/frames.c b/Modules/_remote_debugging/frames.c index 2ace0c0f7676ae..a0b4a1e8a1e542 100644 --- a/Modules/_remote_debugging/frames.c +++ b/Modules/_remote_debugging/frames.c @@ -348,10 +348,12 @@ process_frame_chain( PyObject *extra_frame_info = make_frame_info( unwinder, _Py_LATIN1_CHR('~'), Py_None, extra_frame, Py_None); if (extra_frame_info == NULL) { + Py_XDECREF(frame); return -1; } if (PyList_Append(ctx->frame_info, extra_frame_info) < 0) { Py_DECREF(extra_frame_info); + Py_XDECREF(frame); set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append extra frame"); return -1; } diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index 040bd3db377315..f86bbf8ce5526e 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -420,6 +420,7 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, #if defined(__APPLE__) self->thread_id_offset = 0; + self->thread_id_offset_initialized = 0; #endif #ifdef MS_WINDOWS @@ -583,11 +584,16 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self current_tstate = self->tstate_addr; } + // Acquire main thread state information + uintptr_t main_thread_tstate = GET_MEMBER(uintptr_t, interp_state_buffer, + self->debug_offsets.interpreter_state.threads_main); + while (current_tstate != 0) { uintptr_t prev_tstate = current_tstate; PyObject* frame_info = unwind_stack_for_thread(self, ¤t_tstate, gil_holder_tstate, - gc_frame); + gc_frame, + main_thread_tstate); if (!frame_info) { // Check if this was an intentional skip due to mode-based filtering if ((self->mode == PROFILING_MODE_CPU || self->mode == PROFILING_MODE_GIL || @@ -1207,6 +1213,9 @@ _remote_debugging_exec(PyObject *m) if (PyModule_AddIntConstant(m, "THREAD_STATUS_HAS_EXCEPTION", THREAD_STATUS_HAS_EXCEPTION) < 0) { return -1; } + if (PyModule_AddIntConstant(m, "THREAD_STATUS_MAIN_THREAD", THREAD_STATUS_MAIN_THREAD) < 0) { + return -1; + } if (RemoteDebugging_InitState(st) < 0) { return -1; @@ -1831,6 +1840,7 @@ static PyMethodDef remote_debugging_methods[] = { }; static PyModuleDef_Slot remote_debugging_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _remote_debugging_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_remote_debugging/object_reading.c b/Modules/_remote_debugging/object_reading.c index 447b7fd5926064..59c28e223c545f 100644 --- a/Modules/_remote_debugging/object_reading.c +++ b/Modules/_remote_debugging/object_reading.c @@ -196,6 +196,8 @@ read_py_long( // Validate size: reject garbage (negative or unreasonably large) if (size < 0 || size > MAX_LONG_DIGITS) { + PyErr_Format(PyExc_RuntimeError, + "Invalid PyLong digit count: %zd (expected 0-%d)", size, MAX_LONG_DIGITS); set_exception_cause(unwinder, PyExc_RuntimeError, "Invalid PyLong size (corrupted remote memory)"); return -1; diff --git a/Modules/_remote_debugging/threads.c b/Modules/_remote_debugging/threads.c index 3100b83c8f4899..a38bb945169a77 100644 --- a/Modules/_remote_debugging/threads.c +++ b/Modules/_remote_debugging/threads.c @@ -157,11 +157,11 @@ find_running_frame( int get_thread_status(RemoteUnwinderObject *unwinder, uint64_t tid, uint64_t pthread_id) { #if defined(__APPLE__) && TARGET_OS_OSX - if (unwinder->thread_id_offset == 0) { + if (!unwinder->thread_id_offset_initialized) { uint64_t *tids = (uint64_t *)PyMem_Malloc(MAX_NATIVE_THREADS * sizeof(uint64_t)); if (!tids) { - PyErr_NoMemory(); - return -1; + // Non-fatal: thread status is best-effort + return THREAD_STATE_UNKNOWN; } int n = proc_pidinfo(unwinder->handle.pid, PROC_PIDLISTTHREADS, 0, tids, MAX_NATIVE_THREADS * sizeof(uint64_t)) / sizeof(uint64_t); if (n <= 0) { @@ -176,6 +176,7 @@ get_thread_status(RemoteUnwinderObject *unwinder, uint64_t tid, uint64_t pthread } } unwinder->thread_id_offset = min_offset; + unwinder->thread_id_offset_initialized = 1; PyMem_Free(tids); } struct proc_threadinfo ti; @@ -239,20 +240,21 @@ get_thread_status(RemoteUnwinderObject *unwinder, uint64_t tid, uint64_t pthread unwinder->win_process_buffer_size = n; PVOID new_buffer = PyMem_Realloc(unwinder->win_process_buffer, n); if (!new_buffer) { - return -1; + // Match Linux/macOS: degrade gracefully on alloc failure + return THREAD_STATE_UNKNOWN; } unwinder->win_process_buffer = new_buffer; return get_thread_status(unwinder, tid, pthread_id); } if (status != STATUS_SUCCESS) { - return -1; + return THREAD_STATE_UNKNOWN; } SYSTEM_PROCESS_INFORMATION *pi = (SYSTEM_PROCESS_INFORMATION *)unwinder->win_process_buffer; while ((ULONG)(ULONG_PTR)pi->UniqueProcessId != unwinder->handle.pid) { if (pi->NextEntryOffset == 0) { - // We didn't find the process - return -1; + // Process not found (may have exited) + return THREAD_STATE_UNKNOWN; } pi = (SYSTEM_PROCESS_INFORMATION *)(((BYTE *)pi) + pi->NextEntryOffset); } @@ -264,7 +266,8 @@ get_thread_status(RemoteUnwinderObject *unwinder, uint64_t tid, uint64_t pthread } } - return -1; + // Thread not found (may have exited) + return THREAD_STATE_UNKNOWN; #else return THREAD_STATE_UNKNOWN; #endif @@ -291,7 +294,8 @@ unwind_stack_for_thread( RemoteUnwinderObject *unwinder, uintptr_t *current_tstate, uintptr_t gil_holder_tstate, - uintptr_t gc_frame + uintptr_t gc_frame, + uintptr_t main_thread_tstate ) { PyObject *frame_info = NULL; PyObject *thread_id = NULL; @@ -384,17 +388,21 @@ unwind_stack_for_thread( long pthread_id = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.thread_id); // Optimization: only check CPU status if needed by mode because it's expensive - int cpu_status = -1; + int cpu_status = THREAD_STATE_UNKNOWN; if (unwinder->mode == PROFILING_MODE_CPU || unwinder->mode == PROFILING_MODE_ALL) { cpu_status = get_thread_status(unwinder, tid, pthread_id); } - if (cpu_status == -1) { + if (cpu_status == THREAD_STATE_UNKNOWN) { status_flags |= THREAD_STATUS_UNKNOWN; } else if (cpu_status == THREAD_STATE_RUNNING) { status_flags |= THREAD_STATUS_ON_CPU; } + if (*current_tstate == main_thread_tstate) { + status_flags |= THREAD_STATUS_MAIN_THREAD; + } + // Check if we should skip this thread based on mode int should_skip = 0; if (unwinder->skip_non_matching_threads) { diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index af63271b9fd971..bd44ff31b87c67 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -1059,13 +1059,16 @@ static callback_context * create_callback_context(PyTypeObject *cls, PyObject *callable) { callback_context *ctx = PyMem_Malloc(sizeof(callback_context)); - if (ctx != NULL) { - PyObject *module = PyType_GetModule(cls); - ctx->refcount = 1; - ctx->callable = Py_NewRef(callable); - ctx->module = Py_NewRef(module); - ctx->state = pysqlite_get_state(module); + if (ctx == NULL) { + PyErr_NoMemory(); + return NULL; } + + PyObject *module = PyType_GetModule(cls); + ctx->refcount = 1; + ctx->callable = Py_NewRef(callable); + ctx->module = Py_NewRef(module); + ctx->state = pysqlite_get_state(module); return ctx; } @@ -2198,7 +2201,7 @@ pysqlite_connection_create_collation_impl(pysqlite_Connection *self, * the context before returning. */ if (callable != Py_None) { - free_callback_context(ctx); + decref_callback_context(ctx); } set_error_from_db(self->state, self->db); return NULL; diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 831dd9219f77ab..512d9744d57416 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -778,6 +778,7 @@ module_exec(PyObject *module) } static struct PyModuleDef_Slot module_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index d6cdd861fd85a2..044eb6e5f1fb66 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -43,6 +43,7 @@ static const char copyright[] = #include "pycore_dict.h" // _PyDict_Next() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_tuple.h" // _PyTuple_FromPairSteal #include "pycore_unicodeobject.h" // _PyUnicode_Copy #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() @@ -2572,28 +2573,17 @@ _sre_SRE_Match_end_impl(MatchObject *self, PyObject *group) LOCAL(PyObject*) _pair(Py_ssize_t i1, Py_ssize_t i2) { - PyObject* pair; - PyObject* item; - - pair = PyTuple_New(2); - if (!pair) + PyObject* item1 = PyLong_FromSsize_t(i1); + if (!item1) { return NULL; + } + PyObject* item2 = PyLong_FromSsize_t(i2); + if(!item2) { + Py_DECREF(item1); + return NULL; + } - item = PyLong_FromSsize_t(i1); - if (!item) - goto error; - PyTuple_SET_ITEM(pair, 0, item); - - item = PyLong_FromSsize_t(i2); - if (!item) - goto error; - PyTuple_SET_ITEM(pair, 1, item); - - return pair; - - error: - Py_DECREF(pair); - return NULL; + return _PyTuple_FromPairSteal(item1, item2); } /*[clinic input] @@ -3466,6 +3456,7 @@ sre_exec(PyObject *m) } static PyModuleDef_Slot sre_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, sre_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 2eb31229a9bf3c..4e563379098eaf 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -30,6 +30,7 @@ #include "pycore_long.h" // _PyLong_UnsignedLongLong_Converter() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_time.h" // _PyDeadline_Init() +#include "pycore_tuple.h" // _PyTuple_FromPair /* Include symbols from _socket module */ #include "socketmodule.h" @@ -164,6 +165,17 @@ static void _PySSLFixErrno(void) { #error Unsupported OpenSSL version #endif +#if (OPENSSL_VERSION_NUMBER >= 0x40000000L) +# define OPENSSL_NO_SSL3 +# define OPENSSL_NO_TLS1 +# define OPENSSL_NO_TLS1_1 +# define OPENSSL_NO_TLS1_2 +# define OPENSSL_NO_SSL3_METHOD +# define OPENSSL_NO_TLS1_METHOD +# define OPENSSL_NO_TLS1_1_METHOD +# define OPENSSL_NO_TLS1_2_METHOD +#endif + /* OpenSSL API 1.1.0+ does not include version methods */ #ifndef OPENSSL_NO_SSL3_METHOD extern const SSL_METHOD *SSLv3_method(void); @@ -581,7 +593,7 @@ fill_and_set_sslerror(_sslmodulestate *state, } else { if (PyUnicodeWriter_Format( - writer, "unknown error (0x%x)", errcode) < 0) { + writer, "unknown error (0x%lx)", errcode) < 0) { goto fail; } } @@ -1151,7 +1163,7 @@ _asn1obj2py(_sslmodulestate *state, const ASN1_OBJECT *name, int no_name) static PyObject * _create_tuple_for_attribute(_sslmodulestate *state, - ASN1_OBJECT *name, ASN1_STRING *value) + const ASN1_OBJECT *name, const ASN1_STRING *value) { Py_ssize_t buflen; PyObject *pyattr; @@ -1180,16 +1192,16 @@ _create_tuple_for_attribute(_sslmodulestate *state, } static PyObject * -_create_tuple_for_X509_NAME (_sslmodulestate *state, X509_NAME *xname) +_create_tuple_for_X509_NAME(_sslmodulestate *state, const X509_NAME *xname) { PyObject *dn = NULL; /* tuple which represents the "distinguished name" */ PyObject *rdn = NULL; /* tuple to hold a "relative distinguished name" */ PyObject *rdnt; PyObject *attr = NULL; /* tuple to hold an attribute */ int entry_count = X509_NAME_entry_count(xname); - X509_NAME_ENTRY *entry; - ASN1_OBJECT *name; - ASN1_STRING *value; + const X509_NAME_ENTRY *entry; + const ASN1_OBJECT *name; + const ASN1_STRING *value; int index_counter; int rdn_level = -1; int retcode; @@ -4005,15 +4017,11 @@ _ssl__SSLContext_verify_flags_set_impl(PySSLContext *self, PyObject *value) static int set_min_max_proto_version(PySSLContext *self, PyObject *arg, int what) { - long v; + int v; int result; - if (!PyArg_Parse(arg, "l", &v)) + if (!PyArg_Parse(arg, "i", &v)) return -1; - if (v > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, "Option is too long"); - return -1; - } switch(self->protocol) { case PY_SSL_VERSION_TLS_CLIENT: _Py_FALLTHROUGH; @@ -4048,7 +4056,7 @@ set_min_max_proto_version(PySSLContext *self, PyObject *arg, int what) break; default: PyErr_Format(PyExc_ValueError, - "Unsupported TLS/SSL version 0x%x", v); + "Unsupported TLS/SSL version 0x%x", (unsigned)v); return -1; } @@ -4082,7 +4090,7 @@ set_min_max_proto_version(PySSLContext *self, PyObject *arg, int what) } if (result == 0) { PyErr_Format(PyExc_ValueError, - "Unsupported protocol version 0x%x", v); + "Unsupported protocol version 0x%x", (unsigned)v); return -1; } return 0; @@ -5194,7 +5202,7 @@ _servername_callback(SSL *s, int *al, void *args) return ret; error: - Py_DECREF(ssl_socket); + Py_XDECREF(ssl_socket); *al = SSL_AD_INTERNAL_ERROR; ret = SSL_TLSEXT_ERR_ALERT_FATAL; PyGILState_Release(gstate); @@ -6789,7 +6797,7 @@ do { \ } /* ssl.CertificateError used to be a subclass of ValueError */ - bases = PyTuple_Pack(2, state->PySSLErrorObject, PyExc_ValueError); + bases = _PyTuple_FromPair(state->PySSLErrorObject, PyExc_ValueError); if (bases == NULL) { goto error; } @@ -6967,9 +6975,15 @@ sslmodule_init_constants(PyObject *m) ADD_INT_CONST("PROTOCOL_TLS", PY_SSL_VERSION_TLS); ADD_INT_CONST("PROTOCOL_TLS_CLIENT", PY_SSL_VERSION_TLS_CLIENT); ADD_INT_CONST("PROTOCOL_TLS_SERVER", PY_SSL_VERSION_TLS_SERVER); +#ifndef OPENSSL_NO_TLS1 ADD_INT_CONST("PROTOCOL_TLSv1", PY_SSL_VERSION_TLS1); +#endif +#ifndef OPENSSL_NO_TLS1_1 ADD_INT_CONST("PROTOCOL_TLSv1_1", PY_SSL_VERSION_TLS1_1); +#endif +#ifndef OPENSSL_NO_TLS1_2 ADD_INT_CONST("PROTOCOL_TLSv1_2", PY_SSL_VERSION_TLS1_2); +#endif #define ADD_OPTION(NAME, VALUE) if (sslmodule_add_option(m, NAME, (VALUE)) < 0) return -1 @@ -7312,6 +7326,7 @@ sslmodule_init_lock(PyObject *module) } static PyModuleDef_Slot sslmodule_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, sslmodule_init_types}, {Py_mod_exec, sslmodule_init_exceptions}, {Py_mod_exec, sslmodule_init_socketapi}, diff --git a/Modules/_ssl/cert.c b/Modules/_ssl/cert.c index f2e7be896687c8..061b0fb31716a4 100644 --- a/Modules/_ssl/cert.c +++ b/Modules/_ssl/cert.c @@ -128,7 +128,8 @@ _ssl_Certificate_get_info_impl(PySSLCertificate *self) } static PyObject* -_x509name_print(_sslmodulestate *state, X509_NAME *name, int indent, unsigned long flags) +_x509name_print(_sslmodulestate *state, const X509_NAME *name, + int indent, unsigned long flags) { PyObject *res; BIO *biobuf; diff --git a/Modules/_struct.c b/Modules/_struct.c index 2059218029ea34..c235e27a415543 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -763,14 +763,13 @@ np_halffloat(_structmodulestate *state, char *p, PyObject *v, const formatdef *f static int np_float(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { - float x = (float)PyFloat_AsDouble(v); + double x = PyFloat_AsDouble(v); if (x == -1 && PyErr_Occurred()) { PyErr_SetString(state->StructError, "required argument is not a float"); return -1; } - memcpy(p, &x, sizeof x); - return 0; + return PyFloat_Pack4(x, p, PY_LITTLE_ENDIAN); } static int @@ -1536,10 +1535,6 @@ init_endian_tables(void *Py_UNUSED(arg)) size matches */ if (ptr->size != native->size) break; - /* Skip float and double, could be - "unknown" float format */ - if (ptr->format == 'd' || ptr->format == 'f') - break; /* Skip _Bool, semantics are different for standard size */ if (ptr->format == '?') break; @@ -2881,6 +2876,7 @@ _structmodule_exec(PyObject *m) } static PyModuleDef_Slot _structmodule_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _structmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_suggestions.c b/Modules/_suggestions.c index fb588de78085fe..db1efa7841f995 100644 --- a/Modules/_suggestions.c +++ b/Modules/_suggestions.c @@ -51,6 +51,7 @@ static PyMethodDef module_methods[] = { }; static PyModuleDef_Slot module_slots[] = { + _Py_ABI_SLOT, {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, diff --git a/Modules/_sysconfig.c b/Modules/_sysconfig.c index bcb9d108174f43..ff22739610e794 100644 --- a/Modules/_sysconfig.c +++ b/Modules/_sysconfig.c @@ -127,6 +127,7 @@ static struct PyMethodDef sysconfig_methods[] = { }; static PyModuleDef_Slot sysconfig_slots[] = { + _Py_ABI_SLOT, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index d2e61e9d6acf24..8b6b617aafa427 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -351,7 +351,7 @@ pack_from_list(PyObject *obj, PyObject *items, PyObject *format, item = PySequence_Fast_GET_ITEM(items, i); if ((PyBytes_Check(item) || PyLong_Check(item) || - PyFloat_Check(item)) && nmemb == 1) { + PyFloat_Check(item) || PyComplex_Check(item)) && nmemb == 1) { PyTuple_SET_ITEM(args, 2, item); } else if ((PyList_Check(item) || PyTuple_Check(item)) && @@ -433,7 +433,7 @@ pack_single(char *ptr, PyObject *item, const char *fmt, Py_ssize_t itemsize) PyTuple_SET_ITEM(args, 1, zero); if ((PyBytes_Check(item) || PyLong_Check(item) || - PyFloat_Check(item)) && nmemb == 1) { + PyFloat_Check(item) || PyComplex_Check(item)) && nmemb == 1) { PyTuple_SET_ITEM(args, 2, item); } else if ((PyList_Check(item) || PyTuple_Check(item)) && diff --git a/Modules/_testcapi/immortal.c b/Modules/_testcapi/immortal.c index af510cab655356..1c87025594a48b 100644 --- a/Modules/_testcapi/immortal.c +++ b/Modules/_testcapi/immortal.c @@ -31,13 +31,13 @@ test_immortal_small_ints(PyObject *self, PyObject *Py_UNUSED(ignored)) for (int i = -5; i <= 1024; i++) { PyObject *obj = PyLong_FromLong(i); assert(verify_immortality(obj)); - int has_int_immortal_bit = ((PyLongObject *)obj)->long_value.lv_tag & IMMORTALITY_BIT_MASK; + int has_int_immortal_bit = _PyLong_IsSmallInt((PyLongObject *)obj); assert(has_int_immortal_bit); } for (int i = 1025; i <= 1030; i++) { PyObject *obj = PyLong_FromLong(i); assert(obj); - int has_int_immortal_bit = ((PyLongObject *)obj)->long_value.lv_tag & IMMORTALITY_BIT_MASK; + int has_int_immortal_bit = _PyLong_IsSmallInt((PyLongObject *)obj); assert(!has_int_immortal_bit); Py_DECREF(obj); } diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 6313abf5485fff..008a7d37726869 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -254,6 +254,25 @@ pylongwriter_create(PyObject *module, PyObject *args) } +static PyObject * +pylongwriter_finish_bug(PyObject *module, PyObject *Py_UNUSED(args)) +{ + void *writer_digits; + PyLongWriter *writer = PyLongWriter_Create(0, 3, &writer_digits); + if (writer == NULL) { + return NULL; + } + + assert(PyLong_GetNativeLayout()->digit_size == sizeof(digit)); + digit *digits = writer_digits; + digits[0] = 1; + digits[1] = 1; + // Oops, digits[2] is left uninitialized on purpose + // to test PyLongWriter_Finish() + return PyLongWriter_Finish(writer); +} + + static PyObject * get_pylong_layout(PyObject *module, PyObject *Py_UNUSED(args)) { @@ -271,6 +290,7 @@ static PyMethodDef test_methods[] = { {"pylong_aspid", pylong_aspid, METH_O}, {"pylong_export", pylong_export, METH_O}, {"pylongwriter_create", pylongwriter_create, METH_VARARGS}, + {"pylongwriter_finish_bug", pylongwriter_finish_bug, METH_NOARGS}, {"get_pylong_layout", get_pylong_layout, METH_NOARGS}, {"pylong_ispositive", pylong_ispositive, METH_O}, {"pylong_isnegative", pylong_isnegative, METH_O}, diff --git a/Modules/_testcapi/module.c b/Modules/_testcapi/module.c index 3411b21e942a19..52e1d6d94a3af7 100644 --- a/Modules/_testcapi/module.c +++ b/Modules/_testcapi/module.c @@ -8,6 +8,8 @@ * Lib/test/test_capi/test_module.py */ +PyABIInfo_VAR(abi_info); + static PyObject * module_from_slots_empty(PyObject *self, PyObject *spec) { @@ -17,6 +19,16 @@ module_from_slots_empty(PyObject *self, PyObject *spec) return PyModule_FromSlotsAndSpec(slots, spec); } +static PyObject * +module_from_slots_minimal(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_abi, &abi_info}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + static PyObject * module_from_slots_null(PyObject *self, PyObject *spec) { @@ -27,6 +39,7 @@ static PyObject * module_from_slots_name(PyObject *self, PyObject *spec) { PyModuleDef_Slot slots[] = { + {Py_mod_abi, &abi_info}, {Py_mod_name, "currently ignored..."}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, @@ -39,6 +52,7 @@ static PyObject * module_from_slots_doc(PyObject *self, PyObject *spec) { PyModuleDef_Slot slots[] = { + {Py_mod_abi, &abi_info}, {Py_mod_doc, "the docstring"}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, @@ -51,6 +65,7 @@ static PyObject * module_from_slots_size(PyObject *self, PyObject *spec) { PyModuleDef_Slot slots[] = { + {Py_mod_abi, &abi_info}, {Py_mod_state_size, (void*)123}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, @@ -78,6 +93,7 @@ static PyObject * module_from_slots_methods(PyObject *self, PyObject *spec) { PyModuleDef_Slot slots[] = { + {Py_mod_abi, &abi_info}, {Py_mod_methods, a_methoddef_array}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, @@ -96,6 +112,7 @@ static PyObject * module_from_slots_gc(PyObject *self, PyObject *spec) { PyModuleDef_Slot slots[] = { + {Py_mod_abi, &abi_info}, {Py_mod_state_traverse, noop_traverse}, {Py_mod_state_clear, noop_clear}, {Py_mod_state_free, noop_free}, @@ -128,6 +145,7 @@ static PyObject * module_from_slots_token(PyObject *self, PyObject *spec) { PyModuleDef_Slot slots[] = { + {Py_mod_abi, &abi_info}, {Py_mod_token, (void*)&test_token}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, @@ -156,6 +174,7 @@ static PyObject * module_from_slots_exec(PyObject *self, PyObject *spec) { PyModuleDef_Slot slots[] = { + {Py_mod_abi, &abi_info}, {Py_mod_exec, simple_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, @@ -189,6 +208,7 @@ static PyObject * module_from_slots_create(PyObject *self, PyObject *spec) { PyModuleDef_Slot slots[] = { + {Py_mod_abi, &abi_info}, {Py_mod_create, create_attr_from_spec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, @@ -220,6 +240,7 @@ module_from_slots_repeat_slot(PyObject *self, PyObject *spec) return NULL; } PyModuleDef_Slot slots[] = { + {Py_mod_abi, &abi_info}, {slot_id, "anything"}, {slot_id, "anything else"}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, @@ -238,6 +259,7 @@ module_from_slots_null_slot(PyObject *self, PyObject *spec) } PyModuleDef_Slot slots[] = { {slot_id, NULL}, + {Py_mod_abi, &abi_info}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0}, @@ -254,6 +276,7 @@ module_from_def_slot(PyObject *self, PyObject *spec) } PyModuleDef_Slot slots[] = { {slot_id, "anything"}, + {Py_mod_abi, &abi_info}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0}, @@ -285,6 +308,7 @@ static PyModuleDef parrot_def = { .m_slots = NULL /* set below */, }; static PyModuleDef_Slot parrot_slots[] = { + {Py_mod_abi, &abi_info}, {Py_mod_name, (void*)parrot_name}, {Py_mod_doc, (void*)parrot_doc}, {Py_mod_state_size, (void*)123}, @@ -314,6 +338,43 @@ module_from_def_slot_parrot(PyObject *self, PyObject *spec) return module; } +static PyObject * +module_from_bad_abiinfo(PyObject *self, PyObject *spec) +{ + PyABIInfo bad_abi_info = { + 1, 0, + .abi_version=0x02080000, + }; + PyModuleDef_Slot slots[] = { + {Py_mod_abi, &abi_info}, + {Py_mod_abi, &bad_abi_info}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_multiple_abiinfo(PyObject *self, PyObject *spec) +{ + PyABIInfo extra_abi_info = { + 1, 0, + .flags=PyABIInfo_STABLE | PyABIInfo_FREETHREADING_AGNOSTIC, + .abi_version=0x03040000, + }; + PyModuleDef_Slot slots[] = { + {Py_mod_abi, &abi_info}, + {Py_mod_abi, &abi_info}, + {Py_mod_abi, &extra_abi_info}, + {Py_mod_abi, &extra_abi_info}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + static int another_exec(PyObject *module) { @@ -344,6 +405,7 @@ static PyObject * module_from_def_multiple_exec(PyObject *self, PyObject *spec) { static PyModuleDef_Slot slots[] = { + {Py_mod_abi, &abi_info}, {Py_mod_exec, simple_exec}, {Py_mod_exec, another_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, @@ -399,6 +461,7 @@ pymodule_get_state_size(PyObject *self, PyObject *module) static PyMethodDef test_methods[] = { {"module_from_slots_empty", module_from_slots_empty, METH_O}, + {"module_from_slots_minimal", module_from_slots_minimal, METH_O}, {"module_from_slots_null", module_from_slots_null, METH_O}, {"module_from_slots_name", module_from_slots_name, METH_O}, {"module_from_slots_doc", module_from_slots_doc, METH_O}, @@ -413,6 +476,8 @@ static PyMethodDef test_methods[] = { {"module_from_def_multiple_exec", module_from_def_multiple_exec, METH_O}, {"module_from_def_slot", module_from_def_slot, METH_O}, {"module_from_def_slot_parrot", module_from_def_slot_parrot, METH_O}, + {"module_from_bad_abiinfo", module_from_bad_abiinfo, METH_O}, + {"module_from_multiple_abiinfo", module_from_multiple_abiinfo, METH_O}, {"pymodule_get_token", pymodule_get_token, METH_O}, {"pymodule_get_def", pymodule_get_def, METH_O}, {"pymodule_get_state_size", pymodule_get_state_size, METH_O}, diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index 6d061bb8d51040..5a756a87c15fe9 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -364,7 +364,7 @@ add_code_watcher(PyObject *self, PyObject *which_watcher) watcher_id = PyCode_AddWatcher(error_code_event_handler); } else { - PyErr_Format(PyExc_ValueError, "invalid watcher %d", which_l); + PyErr_Format(PyExc_ValueError, "invalid watcher %ld", which_l); return NULL; } if (watcher_id < 0) { @@ -673,7 +673,7 @@ add_context_watcher(PyObject *self, PyObject *which_watcher) assert(PyLong_Check(which_watcher)); long which_l = PyLong_AsLong(which_watcher); if (which_l < 0 || which_l >= (long)Py_ARRAY_LENGTH(callbacks)) { - PyErr_Format(PyExc_ValueError, "invalid watcher %d", which_l); + PyErr_Format(PyExc_ValueError, "invalid watcher %ld", which_l); return NULL; } int watcher_id = PyContext_AddWatcher(callbacks[which_l]); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index a25b127f1011b8..aa12db20908b97 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -116,8 +116,8 @@ test_sizeof_c_types(PyObject *self, PyObject *Py_UNUSED(ignored)) do { \ if (EXPECTED != sizeof(TYPE)) { \ PyErr_Format(get_testerror(self), \ - "sizeof(%s) = %u instead of %u", \ - #TYPE, sizeof(TYPE), EXPECTED); \ + "sizeof(%s) = %zu instead of %u", \ + #TYPE, sizeof(TYPE), (unsigned)(EXPECTED)); \ return (PyObject*)NULL; \ } \ } while (0) @@ -2607,6 +2607,41 @@ create_managed_weakref_nogc_type(PyObject *self, PyObject *Py_UNUSED(args)) } +static PyObject* +test_soft_deprecated_macros(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + // Test soft-deprecated macros + Py_ALIGNED(64) char buf[4]; + #ifdef __GNUC__ + // Py_ALIGNED must compile everywhere, but only does something + // on "supported" compilers, i.e. GCC + Py_BUILD_ASSERT(__extension__ __alignof__(buf) >= 64); + #endif + assert(strcmp(PY_FORMAT_SIZE_T, "z") == 0); + Py_BUILD_ASSERT(Py_LL(123) == 123LL); + Py_BUILD_ASSERT(sizeof(Py_LL(123)) == sizeof(long long)); + Py_BUILD_ASSERT(sizeof(Py_ULL(123)) == sizeof(unsigned long long)); + Py_BUILD_ASSERT(sizeof(PY_LONG_LONG) == sizeof(long long)); + Py_BUILD_ASSERT(sizeof(PY_INT32_T) == sizeof(int32_t)); + Py_BUILD_ASSERT(sizeof(PY_UINT32_T) == sizeof(uint32_t)); + Py_BUILD_ASSERT(sizeof(PY_INT64_T) == sizeof(int64_t)); + Py_BUILD_ASSERT(sizeof(PY_UINT64_T) == sizeof(uint64_t)); + Py_BUILD_ASSERT(PY_LLONG_MIN == LLONG_MIN); + Py_BUILD_ASSERT(PY_LLONG_MAX == LLONG_MAX); + Py_BUILD_ASSERT(PY_ULLONG_MAX == ULLONG_MAX); + Py_BUILD_ASSERT(PY_SIZE_MAX == SIZE_MAX); + Py_BUILD_ASSERT(PY_LLONG_MIN == LLONG_MIN); + Py_MEMCPY(buf, "abc", 4); + assert(strcmp(buf, "abc") == 0); + Py_BUILD_ASSERT(Py_UNICODE_SIZE == sizeof(wchar_t)); + #ifdef Py_UNICODE_WIDE + Py_BUILD_ASSERT(sizeof(wchar_t) >= 4); + #else + Py_BUILD_ASSERT(sizeof(wchar_t) < 4); + #endif + Py_RETURN_NONE; +} + static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, {"test_config", test_config, METH_NOARGS}, @@ -2704,6 +2739,7 @@ static PyMethodDef TestMethods[] = { {"toggle_reftrace_printer", toggle_reftrace_printer, METH_O}, {"create_managed_weakref_nogc_type", create_managed_weakref_nogc_type, METH_NOARGS}, + {"test_soft_deprecated_macros", test_soft_deprecated_macros, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index e1acce8f586685..c00bad46a54907 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -417,14 +417,14 @@ test_bswap(PyObject *self, PyObject *Py_UNUSED(args)) uint16_t u16 = _Py_bswap16(UINT16_C(0x3412)); if (u16 != UINT16_C(0x1234)) { PyErr_Format(PyExc_AssertionError, - "_Py_bswap16(0x3412) returns %u", u16); + "_Py_bswap16(0x3412) returns %d", u16); return NULL; } uint32_t u32 = _Py_bswap32(UINT32_C(0x78563412)); if (u32 != UINT32_C(0x12345678)) { PyErr_Format(PyExc_AssertionError, - "_Py_bswap32(0x78563412) returns %lu", u32); + "_Py_bswap32(0x78563412) returns %u", u32); return NULL; } @@ -703,7 +703,7 @@ test_edit_cost(PyObject *self, PyObject *Py_UNUSED(args)) static int check_bytes_find(const char *haystack0, const char *needle0, - int offset, Py_ssize_t expected) + Py_ssize_t offset, Py_ssize_t expected) { Py_ssize_t len_haystack = strlen(haystack0); Py_ssize_t len_needle = strlen(needle0); @@ -1158,7 +1158,7 @@ get_interp_settings(PyObject *self, PyObject *args) } else { PyErr_Format(PyExc_NotImplementedError, - "%zd", interpid); + "%d", interpid); return NULL; } assert(interp != NULL); @@ -2838,6 +2838,20 @@ test_threadstate_set_stack_protection(PyObject *self, PyObject *Py_UNUSED(args)) } +static PyObject * +_pyerr_setkeyerror(PyObject *self, PyObject *arg) +{ + // Test that _PyErr_SetKeyError() overrides the current exception + // if an exception is set + PyErr_NoMemory(); + + _PyErr_SetKeyError(arg); + + assert(PyErr_Occurred()); + return NULL; +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_eval_frame_stats", get_eval_frame_stats, METH_NOARGS, NULL}, @@ -2959,6 +2973,7 @@ static PyMethodDef module_functions[] = { {"module_get_gc_hooks", module_get_gc_hooks, METH_O}, {"test_threadstate_set_stack_protection", test_threadstate_set_stack_protection, METH_NOARGS}, + {"_pyerr_setkeyerror", _pyerr_setkeyerror, METH_O}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 6f2d4c89893547..45cbc58b085851 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -2348,17 +2348,9 @@ _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST + // _GUARD_CALLABLE_BUILTIN_FAST { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2370,6 +2362,17 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CALL_BUILTIN_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _Py_BuiltinCallFast_StackRefSteal( @@ -2417,17 +2420,9 @@ _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST_WITH_KEYWORDS + // _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2439,6 +2434,17 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CALL_BUILTIN_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRefSteal(callable, arguments, total_args); @@ -2485,32 +2491,38 @@ _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_BUILTIN_O + // _GUARD_CALLABLE_BUILTIN_O { - args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - if (total_args != 1) { + if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (!PyCFunction_CheckExact(callable_o)) { + if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CALL_BUILTIN_O + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyStackRef_IsNull(self_or_null)) { + args--; + } if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); @@ -3021,31 +3033,44 @@ _PyStackRef value2_st; _PyStackRef value1_st; _PyStackRef res; - value1_st = stack_pointer[-1]; - value2_st = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); - PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); - PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - _PyStackRef tmp = value1_st; - value1_st = PyStackRef_NULL; - stack_pointer[-1] = value1_st; - PyStackRef_CLOSE(tmp); - tmp = value2_st; - value2_st = PyStackRef_NULL; - stack_pointer[-2] = value2_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (res_o == NULL) { - JUMP_TO_LABEL(error); + _PyStackRef vs1; + _PyStackRef vs2; + _PyStackRef value; + // _CALL_INTRINSIC_2 + { + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + vs1 = value1_st; + vs2 = value2_st; + } + // _POP_TOP + { + value = vs2; + stack_pointer[-2] = res; + stack_pointer[-1] = vs1; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = vs1; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -3754,47 +3779,58 @@ _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST + // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (total_args == 0) { + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + if (method->d_method->ml_flags != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_FASTCALL) { + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args == 0) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - assert(self != NULL); + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CALL_METHOD_DESCRIPTOR_FAST + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFast cfunc = _PyCFunctionFast_CAST(method->d_method->ml_meth); PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( callable, - meth, + cfunc, self, arguments, total_args @@ -3839,12 +3875,23 @@ _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (method->d_method->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { @@ -3856,31 +3903,31 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + } + // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; } - PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); assert(self != NULL); - if (!Py_IS_TYPE(self, d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords_CAST(method->d_method->ml_meth); PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( callable, - meth, + cfunc, self, arguments, total_args @@ -3925,49 +3972,60 @@ _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_NOARGS + // _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - if (total_args != 1) { + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + if (method->d_method->ml_flags != METH_NOARGS) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDef *meth = method->d_method; - _PyStackRef self_stackref = args[0]; - PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - if (!Py_IS_TYPE(self, method->d_common.d_type)) { + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (meth->ml_flags != METH_NOARGS) { + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CHECK_RECURSION_LIMIT + { if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CALL_METHOD_DESCRIPTOR_NOARGS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + assert(oparg == 1 || !PyStackRef_IsNull(self_or_null)); + if (!PyStackRef_IsNull(self_or_null)) { + args--; + } + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; + PyCFunction cfunc = method->d_method->ml_meth; _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4022,54 +4080,62 @@ _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_O + // _GUARD_CALLABLE_METHOD_DESCRIPTOR_O { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (total_args != 2) { + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + if (method->d_method->ml_flags != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_O) { + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args != 2) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (_Py_ReachedRecursionLimit(tstate)) { + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - _PyStackRef arg_stackref = arguments[1]; - _PyStackRef self_stackref = arguments[0]; - if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type)) { + } + // _CHECK_RECURSION_LIMIT + { + if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CALL_METHOD_DESCRIPTOR_O + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + } STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; + PyCFunction cfunc = method->d_method->ml_meth; + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + PyObject *arg = PyStackRef_AsPyObjectBorrow(arguments[1]); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, - PyStackRef_AsPyObjectBorrow(self_stackref), - PyStackRef_AsPyObjectBorrow(arg_stackref)); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, arg); stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -4740,13 +4806,16 @@ next_instr += 1; INSTRUCTION_STATS(CLEANUP_THROW); _PyStackRef sub_iter; + _PyStackRef null_in; _PyStackRef last_sent_val; _PyStackRef exc_value_st; _PyStackRef none; + _PyStackRef null_out; _PyStackRef value; exc_value_st = stack_pointer[-1]; last_sent_val = stack_pointer[-2]; - sub_iter = stack_pointer[-3]; + null_in = stack_pointer[-3]; + sub_iter = stack_pointer[-4]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); #if !_Py_TAIL_CALL_INTERP assert(throwflag); @@ -4760,7 +4829,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = sub_iter; sub_iter = value; - stack_pointer[-3] = sub_iter; + stack_pointer[-4] = sub_iter; PyStackRef_CLOSE(tmp); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; @@ -4770,9 +4839,14 @@ last_sent_val = PyStackRef_NULL; stack_pointer[-2] = last_sent_val; PyStackRef_CLOSE(tmp); + tmp = null_in; + null_in = PyStackRef_NULL; + stack_pointer[-3] = null_in; + PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; + stack_pointer += -4; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + null_out = null_in; none = PyStackRef_None; } else { @@ -4782,8 +4856,9 @@ JUMP_TO_LABEL(exception_unwind); } stack_pointer[0] = none; - stack_pointer[1] = value; - stack_pointer += 2; + stack_pointer[1] = null_out; + stack_pointer[2] = value; + stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5526,31 +5601,38 @@ _PyStackRef callable; _PyStackRef dict; _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - callable = stack_pointer[-5 - (oparg - 1)]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_MergeEx(dict_o, update_o, 2); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { + _PyStackRef u; + _PyStackRef value; + // _DICT_MERGE + { + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + PyObject *dupkey = NULL; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatKwargsError(tstate, callable_o, update_o); + int err = _PyDict_MergeUniq(dict_o, update_o, &dupkey); stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatKwargsError(tstate, callable_o, update_o, dupkey); + Py_XDECREF(dupkey); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + u = update; + } + // _POP_TOP + { + value = u; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); } - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -5564,36 +5646,53 @@ INSTRUCTION_STATS(DICT_UPDATE); _PyStackRef dict; _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Update(dict_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { + _PyStackRef upd; + _PyStackRef value; + // _DICT_UPDATE + { + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + int err = PyDict_Update(dict_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { + if (err < 0) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update_o)->tp_name); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *exc = _PyErr_GetRaisedException(tstate); + int has_keys = PyObject_HasAttrWithError(update_o, &_Py_ID(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (has_keys == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'%T' object is not a mapping", + update_o); + Py_DECREF(exc); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_ChainExceptions1(exc); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + JUMP_TO_LABEL(error); } + upd = update; + } + // _POP_TOP + { + value = upd; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); } - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -5668,13 +5767,16 @@ next_instr += 1; INSTRUCTION_STATS(END_SEND); _PyStackRef receiver; + _PyStackRef index_or_null; _PyStackRef value; _PyStackRef val; value = stack_pointer[-1]; - receiver = stack_pointer[-2]; + index_or_null = stack_pointer[-2]; + receiver = stack_pointer[-3]; val = value; - stack_pointer[-2] = val; - stack_pointer += -1; + (void)index_or_null; + stack_pointer[-3] = val; + stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(receiver); @@ -6294,31 +6396,15 @@ _PyStackRef index_or_null; iterable = stack_pointer[-1]; #ifdef Py_STATS - _PyFrame_SetStackPointer(frame, stack_pointer); _Py_GatherStats_GetIter(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); #endif - PyTypeObject *tp = PyStackRef_TYPE(iterable); - if (tp == &PyTuple_Type || tp == &PyList_Type) { - iter = iterable; - index_or_null = PyStackRef_TagInt(0); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - index_or_null = PyStackRef_NULL; - stack_pointer += 1; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef result = _PyEval_GetIter(iterable, &index_or_null, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsError(result)) { + JUMP_TO_LABEL(pop_1_error); } + iter = result; stack_pointer[-1] = iter; stack_pointer[0] = index_or_null; stack_pointer += 1; @@ -6354,51 +6440,6 @@ DISPATCH(); } - TARGET(GET_YIELD_FROM_ITER) { - #if _Py_TAIL_CALL_INTERP - int opcode = GET_YIELD_FROM_ITER; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_YIELD_FROM_ITER); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); - if (PyCoro_CheckExact(iterable_o)) { - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); - } - iter = iterable; - } - else if (PyGen_CheckExact(iterable_o)) { - iter = iterable; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(iterable_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = iterable; - iterable = iter; - stack_pointer[-1] = iterable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer[-1] = iter; - DISPATCH(); - } - TARGET(IMPORT_FROM) { #if _Py_TAIL_CALL_INTERP int opcode = IMPORT_FROM; @@ -7002,10 +7043,12 @@ next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_SEND); _PyStackRef receiver; + _PyStackRef index_or_null; _PyStackRef value; _PyStackRef val; value = stack_pointer[-1]; - receiver = stack_pointer[-2]; + index_or_null = stack_pointer[-2]; + receiver = stack_pointer[-3]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7016,8 +7059,9 @@ } } val = value; - stack_pointer[-2] = val; - stack_pointer += -1; + (void)index_or_null; + stack_pointer[-3] = val; + stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(receiver); @@ -7912,40 +7956,45 @@ INSTRUCTION_STATS(LIST_EXTEND); _PyStackRef list_st; _PyStackRef iterable_st; - iterable_st = stack_pointer[-1]; - list_st = stack_pointer[-2 - (oparg-1)]; - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (none_val == NULL) { + _PyStackRef i; + _PyStackRef value; + // _LIST_EXTEND + { + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { + if (none_val == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + JUMP_TO_LABEL(error); } + assert(Py_IsNone(none_val)); + i = iterable_st; + } + // _POP_TOP + { + value = i; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable_st); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); } - assert(Py_IsNone(none_val)); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable_st); - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -10736,11 +10785,12 @@ _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef receiver; + _PyStackRef null_or_index; _PyStackRef v; _PyStackRef retval; // _SPECIALIZE_SEND { - receiver = stack_pointer[-2]; + receiver = stack_pointer[-3]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION @@ -10758,6 +10808,7 @@ // _SEND { v = stack_pointer[-1]; + null_or_index = stack_pointer[-2]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); PyObject *retval_o; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); @@ -10778,53 +10829,66 @@ gen_frame->previous = frame; DISPATCH_INLINED(gen_frame); } - if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { + if (!PyStackRef_IsNull(null_or_index)) { _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, receiver, &null_or_index); stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + JUMP_TO_LABEL(error); + } + JUMPBY(oparg); + stack_pointer[-2] = null_or_index; + DISPATCH(); + } + retval = item; } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = PyObject_CallMethodOneArg(receiver_o, - &_Py_ID(send), - PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (retval_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { + if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); stack_pointer = _PyFrame_GetStackPointer(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyGen_FetchStopIterationValue(&retval_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err == 0) { - assert(retval_o != NULL); - JUMPBY(oparg); - } else { - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); } + if (retval_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyGen_FetchStopIterationValue(&retval_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err == 0) { + assert(retval_o != NULL); + JUMPBY(oparg); + } + else { + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + } + retval = PyStackRef_FromPyObjectSteal(retval_o); } - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + stack_pointer[-2] = null_or_index; + stack_pointer[-1] = retval; _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); - retval = PyStackRef_FromPyObjectSteal(retval_o); } - stack_pointer[0] = retval; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10855,7 +10919,7 @@ // _SEND_GEN_FRAME { v = stack_pointer[-1]; - receiver = stack_pointer[-2]; + receiver = stack_pointer[-3]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { UPDATE_MISS_STATS(SEND); diff --git a/Modules/_testinternalcapi/test_targets.h b/Modules/_testinternalcapi/test_targets.h index def462bacec176..48fe9c14f4e2dd 100644 --- a/Modules/_testinternalcapi/test_targets.h +++ b/Modules/_testinternalcapi/test_targets.h @@ -16,10 +16,8 @@ static void *opcode_targets_table[256] = { &&TARGET_FORMAT_WITH_SPEC, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, - &&TARGET_GET_ITER, - &&TARGET_RESERVED, &&TARGET_GET_LEN, - &&TARGET_GET_YIELD_FROM_ITER, + &&TARGET_RESERVED, &&TARGET_INTERPRETER_EXIT, &&TARGET_LOAD_BUILD_CLASS, &&TARGET_LOAD_LOCALS, @@ -72,6 +70,7 @@ static void *opcode_targets_table[256] = { &&TARGET_EXTENDED_ARG, &&TARGET_FOR_ITER, &&TARGET_GET_AWAITABLE, + &&TARGET_GET_ITER, &&TARGET_IMPORT_FROM, &&TARGET_IMPORT_NAME, &&TARGET_IS_OP, @@ -128,6 +127,7 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -379,7 +379,7 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&TARGET_TRACE_RECORD, + &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -626,7 +626,6 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); @@ -868,7 +867,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [GET_AWAITABLE] = _TAIL_CALL_GET_AWAITABLE, [GET_ITER] = _TAIL_CALL_GET_ITER, [GET_LEN] = _TAIL_CALL_GET_LEN, - [GET_YIELD_FROM_ITER] = _TAIL_CALL_GET_YIELD_FROM_ITER, [IMPORT_FROM] = _TAIL_CALL_IMPORT_FROM, [IMPORT_NAME] = _TAIL_CALL_IMPORT_NAME, [INSTRUMENTED_CALL] = _TAIL_CALL_INSTRUMENTED_CALL, @@ -1002,6 +1000,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, + [120] = _TAIL_CALL_UNKNOWN_OPCODE, [121] = _TAIL_CALL_UNKNOWN_OPCODE, [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1126,7 +1125,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [GET_AWAITABLE] = _TAIL_CALL_TRACE_RECORD, [GET_ITER] = _TAIL_CALL_TRACE_RECORD, [GET_LEN] = _TAIL_CALL_TRACE_RECORD, - [GET_YIELD_FROM_ITER] = _TAIL_CALL_TRACE_RECORD, [IMPORT_FROM] = _TAIL_CALL_TRACE_RECORD, [IMPORT_NAME] = _TAIL_CALL_TRACE_RECORD, [INSTRUMENTED_CALL] = _TAIL_CALL_TRACE_RECORD, @@ -1260,6 +1258,7 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACE_RECORD, [WITH_EXCEPT_START] = _TAIL_CALL_TRACE_RECORD, [YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, + [120] = _TAIL_CALL_UNKNOWN_OPCODE, [121] = _TAIL_CALL_UNKNOWN_OPCODE, [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index e286eaae820b2b..54f53c899f5e39 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -435,6 +435,7 @@ static int execfunc(PyObject *m) } static PyModuleDef_Slot main_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, @@ -481,6 +482,7 @@ createfunc_nonmodule(PyObject *spec, PyModuleDef *def) } static PyModuleDef_Slot slots_create_nonmodule[] = { + _Py_ABI_SLOT, {Py_mod_create, createfunc_nonmodule}, {0, NULL}, }; @@ -527,6 +529,7 @@ PyInit__testmultiphase_nonmodule_with_methods(void) /**** Non-ASCII-named modules ****/ static PyModuleDef_Slot nonascii_slots[] = { + _Py_ABI_SLOT, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -689,6 +692,7 @@ createfunc_noop(PyObject *spec, PyModuleDef *def) } static PyModuleDef_Slot slots_multiple_create_slots[] = { + _Py_ABI_SLOT, {Py_mod_create, createfunc_noop}, {Py_mod_create, createfunc_noop}, {0, NULL}, @@ -710,6 +714,7 @@ createfunc_null(PyObject *spec, PyModuleDef *def) } static PyModuleDef_Slot slots_create_null[] = { + _Py_ABI_SLOT, {Py_mod_create, createfunc_null}, {0, NULL}, }; @@ -752,6 +757,7 @@ createfunc_unreported_exception(PyObject *spec, PyModuleDef *def) } static PyModuleDef_Slot slots_create_unreported_exception[] = { + _Py_ABI_SLOT, {Py_mod_create, createfunc_unreported_exception}, {0, NULL}, }; @@ -766,6 +772,7 @@ PyInit__testmultiphase_create_unreported_exception(void) } static PyModuleDef_Slot slots_nonmodule_with_exec_slots[] = { + _Py_ABI_SLOT, {Py_mod_create, createfunc_nonmodule}, {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, @@ -789,6 +796,7 @@ execfunc_err(PyObject *mod) } static PyModuleDef_Slot slots_exec_err[] = { + _Py_ABI_SLOT, {Py_mod_exec, execfunc_err}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, @@ -812,6 +820,7 @@ execfunc_raise(PyObject *spec) } static PyModuleDef_Slot slots_exec_raise[] = { + _Py_ABI_SLOT, {Py_mod_exec, execfunc_raise}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, @@ -835,6 +844,7 @@ execfunc_unreported_exception(PyObject *mod) } static PyModuleDef_Slot slots_exec_unreported_exception[] = { + _Py_ABI_SLOT, {Py_mod_exec, execfunc_unreported_exception}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, @@ -893,6 +903,7 @@ meth_state_access_exec(PyObject *m) } static PyModuleDef_Slot meth_state_access_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, meth_state_access_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, @@ -943,6 +954,7 @@ PyInit__test_module_state_shared(void) /* multiple interpreters support */ static PyModuleDef_Slot slots_multiple_multiple_interpreters_slots[] = { + _Py_ABI_SLOT, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, @@ -961,6 +973,7 @@ PyInit__testmultiphase_multiple_multiple_interpreters_slots(void) } static PyModuleDef_Slot non_isolated_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, @@ -979,6 +992,7 @@ PyInit__test_non_isolated(void) static PyModuleDef_Slot shared_gil_only_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, execfunc}, /* Note that Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED is the default. We put it here explicitly to draw attention to the contrast @@ -1000,6 +1014,7 @@ PyInit__test_shared_gil_only(void) static PyModuleDef_Slot no_multiple_interpreter_slot_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, execfunc}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, @@ -1019,10 +1034,13 @@ PyInit__test_no_multiple_interpreter_slot(void) /* PyModExport_* hooks */ +PyABIInfo_VAR(abi_info); + PyMODEXPORT_FUNC PyModExport__test_from_modexport(void) { static PyModuleDef_Slot slots[] = { + {Py_mod_abi, &abi_info}, {Py_mod_name, "_test_from_modexport"}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, @@ -1035,6 +1053,7 @@ PyMODEXPORT_FUNC PyModExport__test_from_modexport_gil_used(void) { static PyModuleDef_Slot slots[] = { + {Py_mod_abi, &abi_info}, {Py_mod_name, "_test_from_modexport_gil_used"}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_USED}, @@ -1085,6 +1104,7 @@ PyMODEXPORT_FUNC PyModExport__test_from_modexport_create_nonmodule(void) { static PyModuleDef_Slot slots[] = { + {Py_mod_abi, &abi_info}, {Py_mod_name, "_test_from_modexport_create_nonmodule"}, {Py_mod_create, modexport_create_string}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, @@ -1098,6 +1118,7 @@ PyMODEXPORT_FUNC PyModExport__test_from_modexport_create_nonmodule_gil_used(void) { static PyModuleDef_Slot slots[] = { + {Py_mod_abi, &abi_info}, {Py_mod_name, "_test_from_modexport_create_nonmodule"}, {Py_mod_create, modexport_create_string}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, @@ -1117,6 +1138,18 @@ PyModExport__test_from_modexport_empty_slots(void) return modexport_empty_slots; } + +static PyModuleDef_Slot modexport_minimal_slots[] = { + {Py_mod_abi, &abi_info}, + {0}, +}; + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_minimal_slots(void) +{ + return modexport_minimal_slots; +} + static int modexport_smoke_exec(PyObject *mod) { @@ -1157,13 +1190,13 @@ modexport_smoke_get_test_token(PyObject *mod, PyObject *arg) } static PyObject * -modexport_get_empty_slots(PyObject *mod, PyObject *arg) +modexport_get_minimal_slots(PyObject *mod, PyObject *arg) { /* Get the address of modexport_empty_slots. - * This method would be in the `_test_from_modexport_empty_slots` module, + * This method would be in the `_test_from_modexport_minimal_slots` module, * if it had a methods slot. */ - return PyLong_FromVoidPtr(&modexport_empty_slots); + return PyLong_FromVoidPtr(&modexport_minimal_slots); } static void @@ -1183,10 +1216,11 @@ PyModExport__test_from_modexport_smoke(void) static PyMethodDef methods[] = { {"get_state_int", modexport_smoke_get_state_int, METH_NOARGS}, {"get_test_token", modexport_smoke_get_test_token, METH_NOARGS}, - {"get_modexport_empty_slots", modexport_get_empty_slots, METH_NOARGS}, + {"get_modexport_minimal_slots", modexport_get_minimal_slots, METH_NOARGS}, {0}, }; static PyModuleDef_Slot slots[] = { + {Py_mod_abi, &abi_info}, {Py_mod_name, "_test_from_modexport_smoke"}, {Py_mod_doc, "the expected docstring"}, {Py_mod_exec, modexport_smoke_exec}, diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 73eff27343618c..135b53111014d1 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -11,6 +11,7 @@ #include "pycore_pylifecycle.h" #include "pycore_pystate.h" // _PyThreadState_SetCurrent() #include "pycore_time.h" // _PyTime_FromSeconds() +#include "pycore_tuple.h" // _PyTuple_FromPairSteal #include "pycore_weakref.h" // _PyWeakref_GET_REF() #include // offsetof() @@ -1480,13 +1481,11 @@ create_sentinel_wr(localobject *self) return NULL; } - PyObject *args = PyTuple_New(2); + PyObject *args = _PyTuple_FromPairSteal(self_wr, + Py_NewRef(tstate->threading_local_key)); if (args == NULL) { - Py_DECREF(self_wr); return NULL; } - PyTuple_SET_ITEM(args, 0, self_wr); - PyTuple_SET_ITEM(args, 1, Py_NewRef(tstate->threading_local_key)); PyObject *cb = PyCFunction_New(&wr_callback_def, args); Py_DECREF(args); @@ -2857,6 +2856,7 @@ PyDoc_STRVAR(thread_doc, The 'threading' module provides a more convenient interface."); static PyModuleDef_Slot thread_module_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, thread_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 1524d02d9e5a5e..bbe2a428454e0c 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -3476,6 +3476,11 @@ static struct PyModuleDef _tkintermodule = { PyMODINIT_FUNC PyInit__tkinter(void) { + PyABIInfo_VAR(abi_info); + if (PyABIInfo_Check(&abi_info, "_tkinter") < 0) { + return NULL; + } + PyObject *m, *uexe, *cexe; tcl_lock = PyThread_allocate_lock(); diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 21baa6ea003884..56d83ea0dcb2a7 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -216,6 +216,11 @@ static struct PyModuleDef module_def = { PyMODINIT_FUNC PyInit__tracemalloc(void) { + PyABIInfo_VAR(abi_info); + if (PyABIInfo_Check(&abi_info, "_tracemalloc") < 0) { + return NULL; + } + PyObject *mod = PyModule_Create(&module_def); if (mod == NULL) { return NULL; diff --git a/Modules/_typesmodule.c b/Modules/_typesmodule.c index 6c9e7a0a3ba053..232c6cd46d1aa8 100644 --- a/Modules/_typesmodule.c +++ b/Modules/_typesmodule.c @@ -54,6 +54,7 @@ _types_exec(PyObject *m) } static struct PyModuleDef_Slot _typesmodule_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _types_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_typingmodule.c b/Modules/_typingmodule.c index e51279c808a2e1..9f698d3e48d5f0 100644 --- a/Modules/_typingmodule.c +++ b/Modules/_typingmodule.c @@ -74,6 +74,7 @@ _typing_exec(PyObject *m) } static struct PyModuleDef_Slot _typingmodule_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _typing_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_weakref.c b/Modules/_weakref.c index ecaa08ff60f203..623252728554f7 100644 --- a/Modules/_weakref.c +++ b/Modules/_weakref.c @@ -162,6 +162,7 @@ weakref_exec(PyObject *module) } static struct PyModuleDef_Slot weakref_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, weakref_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 985706737c5a36..ffa407b2f21f73 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -3328,6 +3328,7 @@ static int winapi_exec(PyObject *m) } static PyModuleDef_Slot winapi_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, winapi_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_xxtestfuzz/_xxtestfuzz.c b/Modules/_xxtestfuzz/_xxtestfuzz.c index 0e0ca5f95fa449..a2f01eb2490135 100644 --- a/Modules/_xxtestfuzz/_xxtestfuzz.c +++ b/Modules/_xxtestfuzz/_xxtestfuzz.c @@ -28,7 +28,10 @@ static PyMethodDef module_methods[] = { {NULL}, }; +PyABIInfo_VAR(abi_info); + static PyModuleDef_Slot module_slots[] = { + {Py_mod_abi, &abi_info}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index f37f195735b67e..eaffd020ed97c0 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -991,7 +991,7 @@ load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) } if (!PyTuple_CheckExact(data_tuple)) { - PyErr_Format(PyExc_TypeError, "Invalid data result type: %r", + PyErr_Format(PyExc_TypeError, "Invalid data result type: %R", data_tuple); goto error; } @@ -1075,7 +1075,7 @@ load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) } trans_idx[i] = (size_t)cur_trans_idx; - if (trans_idx[i] > self->num_ttinfos) { + if (trans_idx[i] >= self->num_ttinfos) { PyErr_Format( PyExc_ValueError, "Invalid transition index found while reading TZif: %zd", @@ -2081,7 +2081,7 @@ utcoff_to_dstoff(size_t *trans_idx, long *utcoffs, long *dstoffs, dstoff = utcoff - utcoffs[comp_idx]; } - if (!dstoff && idx < (num_ttinfos - 1)) { + if (!dstoff && idx < (num_ttinfos - 1) && i + 1 < num_transitions) { comp_idx = trans_idx[i + 1]; // If the following transition is also DST and we couldn't find @@ -2795,6 +2795,7 @@ zoneinfomodule_exec(PyObject *m) } static PyModuleDef_Slot zoneinfomodule_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, zoneinfomodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index 25ededd03a380a..94246dd93b17de 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -744,6 +744,7 @@ _zstd_free(void *module) } static struct PyModuleDef_Slot _zstd_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _zstd_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index 13071b7a2bacf0..0186ee92f5b147 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -101,7 +101,7 @@ _zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options) /* Check key type */ if (Py_TYPE(key) == mod_state->CParameter_type) { PyErr_SetString(PyExc_TypeError, - "compression options dictionary key must not be a " + "decompression options dictionary key must not be a " "CompressionParameter attribute"); return -1; } diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index ec6a9840131e4d..a86a7561271b87 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -11,8 +11,10 @@ #include "pycore_bytesobject.h" // _PyBytes_Repeat #include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_floatobject.h" // _PY_FLOAT_BIG_ENDIAN #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_tuple.h" // _PyTuple_FromPairSteal #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include // offsetof() @@ -91,9 +93,6 @@ enum machine_format_code { * instead of using the memory content of the array directly. In that * case, the array_reconstructor mechanism is bypassed completely, and * the standard array constructor is used instead. - * - * This is will most likely occur when the machine doesn't use IEEE - * floating-point numbers. */ UNSIGNED_INT8 = 0, @@ -117,10 +116,16 @@ enum machine_format_code { UTF16_LE = 18, UTF16_BE = 19, UTF32_LE = 20, - UTF32_BE = 21 + UTF32_BE = 21, + IEEE_754_FLOAT_COMPLEX_LE = 22, + IEEE_754_FLOAT_COMPLEX_BE = 23, + IEEE_754_DOUBLE_COMPLEX_LE = 24, + IEEE_754_DOUBLE_COMPLEX_BE = 25, + IEEE_754_FLOAT16_LE = 26, + IEEE_754_FLOAT16_BE = 27 }; #define MACHINE_FORMAT_CODE_MIN 0 -#define MACHINE_FORMAT_CODE_MAX 21 +#define MACHINE_FORMAT_CODE_MAX 27 /* @@ -609,6 +614,32 @@ QQ_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) return 0; } +static PyObject * +e_getitem(arrayobject *ap, Py_ssize_t i) +{ + double x = PyFloat_Unpack2(ap->ob_item + sizeof(short)*i, + PY_LITTLE_ENDIAN); + + return PyFloat_FromDouble(x); +} + +static int +e_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +{ + float x; + if (!PyArg_Parse(v, "f;array item must be float", &x)) { + return -1; + } + + CHECK_ARRAY_BOUNDS(ap, i); + + if (i >= 0) { + return PyFloat_Pack2(x, ap->ob_item + sizeof(short)*i, + PY_LITTLE_ENDIAN); + } + return 0; +} + static PyObject * f_getitem(arrayobject *ap, Py_ssize_t i) { @@ -649,6 +680,64 @@ d_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) return 0; } +static PyObject * +cf_getitem(arrayobject *ap, Py_ssize_t i) +{ + float f[2]; + + memcpy(&f, ap->ob_item + i*sizeof(f), sizeof(f)); + return PyComplex_FromDoubles(f[0], f[1]); +} + +static int +cf_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +{ + Py_complex x; + float f[2]; + + if (!PyArg_Parse(v, "D;array item must be complex", &x)) { + return -1; + } + + CHECK_ARRAY_BOUNDS(ap, i); + + f[0] = (float)x.real; + f[1] = (float)x.imag; + if (i >= 0) { + memcpy(ap->ob_item + i*sizeof(f), &f, sizeof(f)); + } + return 0; +} + +static PyObject * +cd_getitem(arrayobject *ap, Py_ssize_t i) +{ + double f[2]; + + memcpy(&f, ap->ob_item + i*sizeof(f), sizeof(f)); + return PyComplex_FromDoubles(f[0], f[1]); +} + +static int +cd_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +{ + Py_complex x; + double f[2]; + + if (!PyArg_Parse(v, "D;array item must be complex", &x)) { + return -1; + } + + CHECK_ARRAY_BOUNDS(ap, i); + + f[0] = x.real; + f[1] = x.imag; + if (i >= 0) { + memcpy(ap->ob_item + i*sizeof(f), &f, sizeof(f)); + } + return 0; +} + #define DEFINE_COMPAREITEMS(code, type) \ static int \ code##_compareitems(const void *lhs, const void *rhs, Py_ssize_t length) \ @@ -691,8 +780,11 @@ static const struct arraydescr descriptors[] = { {'L', sizeof(long), LL_getitem, LL_setitem, LL_compareitems, "L", 1, 0}, {'q', sizeof(long long), q_getitem, q_setitem, q_compareitems, "q", 1, 1}, {'Q', sizeof(long long), QQ_getitem, QQ_setitem, QQ_compareitems, "Q", 1, 0}, + {'e', sizeof(short), e_getitem, e_setitem, NULL, "e", 0, 0}, {'f', sizeof(float), f_getitem, f_setitem, NULL, "f", 0, 0}, {'d', sizeof(double), d_getitem, d_setitem, NULL, "d", 0, 0}, + {'F', 2*sizeof(float), cf_getitem, cf_setitem, NULL, "F", 0, 0}, + {'D', 2*sizeof(double), cd_getitem, cd_setitem, NULL, "D", 0, 0}, {'\0', 0, 0, 0, 0, 0, 0} /* Sentinel */ }; @@ -1451,27 +1543,17 @@ static PyObject * array_array_buffer_info_impl(arrayobject *self) /*[clinic end generated code: output=9b2a4ec3ae7e98e7 input=63d9ad83ba60cda8]*/ { - PyObject *retval = NULL, *v; - - retval = PyTuple_New(2); - if (!retval) - return NULL; - - v = PyLong_FromVoidPtr(self->ob_item); - if (v == NULL) { - Py_DECREF(retval); + PyObject* item1 = PyLong_FromVoidPtr(self->ob_item); + if (item1 == NULL) { return NULL; } - PyTuple_SET_ITEM(retval, 0, v); - - v = PyLong_FromSsize_t(Py_SIZE(self)); - if (v == NULL) { - Py_DECREF(retval); + PyObject* item2 = PyLong_FromSsize_t(Py_SIZE(self)); + if (item2 == NULL) { + Py_DECREF(item1); return NULL; } - PyTuple_SET_ITEM(retval, 1, v); - return retval; + return _PyTuple_FromPairSteal(item1, item2); } /*[clinic input] @@ -1496,13 +1578,14 @@ array.array.byteswap Byteswap all items of the array. -If the items in the array are not 1, 2, 4, or 8 bytes in size, RuntimeError is -raised. +If the items in the array are not 1, 2, 4, 8 or 16 bytes in size, RuntimeError +is raised. Note, that for complex types the order of +components (the real part, followed by imaginary part) is preserved. [clinic start generated code]*/ static PyObject * array_array_byteswap_impl(arrayobject *self) -/*[clinic end generated code: output=5f8236cbdf0d90b5 input=9af1d1749000b14f]*/ +/*[clinic end generated code: output=5f8236cbdf0d90b5 input=aafda275f48191d0]*/ { char *p; Py_ssize_t i; @@ -1528,19 +1611,66 @@ array_array_byteswap_impl(arrayobject *self) } break; case 8: + if (self->ob_descr->typecode != 'F') { + for (p = self->ob_item, i = Py_SIZE(self); --i >= 0; p += 8) { + char p0 = p[0]; + char p1 = p[1]; + char p2 = p[2]; + char p3 = p[3]; + p[0] = p[7]; + p[1] = p[6]; + p[2] = p[5]; + p[3] = p[4]; + p[4] = p3; + p[5] = p2; + p[6] = p1; + p[7] = p0; + } + } + else { + for (p = self->ob_item, i = Py_SIZE(self); --i >= 0; p += 8) { + char t0 = p[0]; + char t1 = p[1]; + p[0] = p[3]; + p[1] = p[2]; + p[2] = t1; + p[3] = t0; + t0 = p[4]; + t1 = p[5]; + p[4] = p[7]; + p[5] = p[6]; + p[6] = t1; + p[7] = t0; + } + } + break; + case 16: + assert(self->ob_descr->typecode == 'D'); for (p = self->ob_item, i = Py_SIZE(self); --i >= 0; p += 8) { - char p0 = p[0]; - char p1 = p[1]; - char p2 = p[2]; - char p3 = p[3]; + char t0 = p[0]; + char t1 = p[1]; + char t2 = p[2]; + char t3 = p[3]; p[0] = p[7]; p[1] = p[6]; p[2] = p[5]; p[3] = p[4]; - p[4] = p3; - p[5] = p2; - p[6] = p1; - p[7] = p0; + p[4] = t3; + p[5] = t2; + p[6] = t1; + p[7] = t0; + t0 = p[8]; + t1 = p[9]; + t2 = p[10]; + t3 = p[11]; + p[8] = p[15]; + p[9] = p[14]; + p[10] = p[13]; + p[11] = p[12]; + p[12] = t3; + p[13] = t2; + p[14] = t1; + p[15] = t0; } break; default: @@ -1975,7 +2105,13 @@ static const struct mformatdescr { {4, 0, 0}, /* 18: UTF16_LE */ {4, 0, 1}, /* 19: UTF16_BE */ {8, 0, 0}, /* 20: UTF32_LE */ - {8, 0, 1} /* 21: UTF32_BE */ + {8, 0, 1}, /* 21: UTF32_BE */ + {8, 0, 0}, /* 22: IEEE_754_FLOAT_COMPLEX_LE */ + {8, 0, 1}, /* 23: IEEE_754_FLOAT_COMPLEX_BE */ + {16, 0, 0}, /* 24: IEEE_754_DOUBLE_COMPLEX_LE */ + {16, 0, 1}, /* 25: IEEE_754_DOUBLE_COMPLEX_BE */ + {2, 0, 0}, /* 26: IEEE_754_FLOAT16_LE */ + {2, 0, 1} /* 27: IEEE_754_FLOAT16_BE */ }; @@ -2010,25 +2146,22 @@ typecode_to_mformat_code(char typecode) case 'w': return UTF32_LE + is_big_endian; + case 'e': + return _PY_FLOAT_BIG_ENDIAN ? IEEE_754_FLOAT16_BE : IEEE_754_FLOAT16_LE; + case 'f': - if (sizeof(float) == 4) { - const float y = 16711938.0; - if (memcmp(&y, "\x4b\x7f\x01\x02", 4) == 0) - return IEEE_754_FLOAT_BE; - if (memcmp(&y, "\x02\x01\x7f\x4b", 4) == 0) - return IEEE_754_FLOAT_LE; - } - return UNKNOWN_FORMAT; + return _PY_FLOAT_BIG_ENDIAN ? IEEE_754_FLOAT_BE : IEEE_754_FLOAT_LE; case 'd': - if (sizeof(double) == 8) { - const double x = 9006104071832581.0; - if (memcmp(&x, "\x43\x3f\xff\x01\x02\x03\x04\x05", 8) == 0) - return IEEE_754_DOUBLE_BE; - if (memcmp(&x, "\x05\x04\x03\x02\x01\xff\x3f\x43", 8) == 0) - return IEEE_754_DOUBLE_LE; - } - return UNKNOWN_FORMAT; + return _PY_FLOAT_BIG_ENDIAN ? IEEE_754_DOUBLE_BE : IEEE_754_DOUBLE_LE; + + case 'F': + return _PY_FLOAT_BIG_ENDIAN ? \ + IEEE_754_FLOAT_COMPLEX_BE : IEEE_754_FLOAT_COMPLEX_LE; + + case 'D': + return _PY_FLOAT_BIG_ENDIAN ? \ + IEEE_754_DOUBLE_COMPLEX_BE : IEEE_754_DOUBLE_COMPLEX_LE; /* Integers */ case 'h': @@ -2106,13 +2239,10 @@ make_array(PyTypeObject *arraytype, char typecode, PyObject *items) if (typecode_obj == NULL) return NULL; - new_args = PyTuple_New(2); + new_args = _PyTuple_FromPairSteal(typecode_obj, Py_NewRef(items)); if (new_args == NULL) { - Py_DECREF(typecode_obj); return NULL; } - PyTuple_SET_ITEM(new_args, 0, typecode_obj); - PyTuple_SET_ITEM(new_args, 1, Py_NewRef(items)); array_obj = array_new(arraytype, new_args, NULL); Py_DECREF(new_args); @@ -2201,6 +2331,27 @@ array__array_reconstructor_impl(PyObject *module, PyTypeObject *arraytype, return NULL; } switch (mformat_code) { + case IEEE_754_FLOAT16_LE: + case IEEE_754_FLOAT16_BE: { + Py_ssize_t i; + int le = (mformat_code == IEEE_754_FLOAT_LE) ? 1 : 0; + Py_ssize_t itemcount = Py_SIZE(items) / 2; + const char *memstr = PyBytes_AS_STRING(items); + + converted_items = PyList_New(itemcount); + if (converted_items == NULL) + return NULL; + for (i = 0; i < itemcount; i++) { + PyObject *pyfloat = PyFloat_FromDouble( + PyFloat_Unpack2(&memstr[i * 2], le)); + if (pyfloat == NULL) { + Py_DECREF(converted_items); + return NULL; + } + PyList_SET_ITEM(converted_items, i, pyfloat); + } + break; + } case IEEE_754_FLOAT_LE: case IEEE_754_FLOAT_BE: { Py_ssize_t i; @@ -2243,6 +2394,52 @@ array__array_reconstructor_impl(PyObject *module, PyTypeObject *arraytype, } break; } + case IEEE_754_FLOAT_COMPLEX_LE: + case IEEE_754_FLOAT_COMPLEX_BE: { + Py_ssize_t i; + int le = (mformat_code == IEEE_754_FLOAT_COMPLEX_LE) ? 1 : 0; + Py_ssize_t itemcount = Py_SIZE(items) / 8; + const char *memstr = PyBytes_AS_STRING(items); + + converted_items = PyList_New(itemcount); + if (converted_items == NULL) { + return NULL; + } + for (i = 0; i < itemcount; i++) { + PyObject *pycomplex = PyComplex_FromDoubles( + PyFloat_Unpack4(&memstr[i * 8], le), + PyFloat_Unpack4(&memstr[i * 8] + 4, le)); + if (pycomplex == NULL) { + Py_DECREF(converted_items); + return NULL; + } + PyList_SET_ITEM(converted_items, i, pycomplex); + } + break; + } + case IEEE_754_DOUBLE_COMPLEX_LE: + case IEEE_754_DOUBLE_COMPLEX_BE: { + Py_ssize_t i; + int le = (mformat_code == IEEE_754_DOUBLE_COMPLEX_LE) ? 1 : 0; + Py_ssize_t itemcount = Py_SIZE(items) / 16; + const char *memstr = PyBytes_AS_STRING(items); + + converted_items = PyList_New(itemcount); + if (converted_items == NULL) { + return NULL; + } + for (i = 0; i < itemcount; i++) { + PyObject *pycomplex = PyComplex_FromDoubles( + PyFloat_Unpack8(&memstr[i * 16], le), + PyFloat_Unpack8(&memstr[i * 16] + 8, le)); + if (pycomplex == NULL) { + Py_DECREF(converted_items); + return NULL; + } + PyList_SET_ITEM(converted_items, i, pycomplex); + } + break; + } case UTF16_LE: case UTF16_BE: { int byteorder = (mformat_code == UTF16_LE) ? -1 : 1; @@ -2760,11 +2957,9 @@ array_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) view->internal = NULL; if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) { view->format = (char *)self->ob_descr->formats; -#ifdef Py_UNICODE_WIDE - if (self->ob_descr->typecode == 'u') { + if (sizeof(wchar_t) >= 4 && self->ob_descr->typecode == 'u') { view->format = "w"; } -#endif } self->ob_exports++; @@ -2950,7 +3145,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyDoc_STRVAR(module_doc, "This module defines an object type which can efficiently represent\n\ an array of basic values: characters, integers, floating-point\n\ -numbers. Arrays are sequence types and behave very much like lists,\n\ +numbers, complex numbers. Arrays are sequence types and behave very much like lists,\n\ except that the type of objects stored in them is constrained.\n"); PyDoc_STRVAR(arraytype_doc, @@ -2977,8 +3172,11 @@ The following type codes are defined:\n\ 'L' unsigned integer 4\n\ 'q' signed integer 8 (see note)\n\ 'Q' unsigned integer 8 (see note)\n\ + 'e' 16-bit IEEE floats 2\n\ 'f' floating-point 4\n\ 'd' floating-point 8\n\ + 'F' float complex 8\n\ + 'D' double complex 16\n\ \n\ NOTE: The 'u' typecode corresponds to Python's unicode character. On\n\ narrow builds this is 2-bytes on wide builds this is 4-bytes.\n\ @@ -3326,6 +3524,7 @@ array_modexec(PyObject *m) } static PyModuleDef_Slot arrayslots[] = { + _Py_ABI_SLOT, {Py_mod_exec, array_modexec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index 3ddbbd59a1ef0c..177b09d3dafbd9 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -341,6 +341,7 @@ Two public functions, register and unregister, are defined.\n\ "); static PyModuleDef_Slot atexitmodule_slots[] = { + _Py_ABI_SLOT, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} diff --git a/Modules/binascii.c b/Modules/binascii.c index f85f32b32e962c..9193137877aef9 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -78,7 +78,7 @@ get_binascii_state(PyObject *module) /* Align to 64 bytes for L1 cache line friendliness */ -static const unsigned char table_a2b_base64[] Py_ALIGNED(64) = { +static const _Py_ALIGNED_DEF(64, unsigned char) table_a2b_base64[] = { -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,62, -1,-1,-1,63, @@ -110,7 +110,7 @@ static const unsigned char table_a2b_base64[] Py_ALIGNED(64) = { */ /* Align to 64 bytes for L1 cache line friendliness */ -static const unsigned char table_b2a_base64[] Py_ALIGNED(64) = +static const _Py_ALIGNED_DEF(64, unsigned char) table_b2a_base64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; /* Encode 3 bytes into 4 base64 characters. */ @@ -189,7 +189,7 @@ base64_decode_fast(const unsigned char *in, Py_ssize_t in_len, } -static const unsigned char table_a2b_base85[] Py_ALIGNED(64) = { +static const _Py_ALIGNED_DEF(64, unsigned char) table_a2b_base85[] = { -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,62,-1,63, 64,65,66,-1, 67,68,69,70, -1,71,-1,-1, @@ -209,7 +209,7 @@ static const unsigned char table_a2b_base85[] Py_ALIGNED(64) = { -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, }; -static const unsigned char table_a2b_base85_a85[] Py_ALIGNED(64) = { +static const _Py_ALIGNED_DEF(64, unsigned char) table_a2b_base85_a85[] = { -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14, @@ -229,11 +229,11 @@ static const unsigned char table_a2b_base85_a85[] Py_ALIGNED(64) = { -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, }; -static const unsigned char table_b2a_base85[] Py_ALIGNED(64) = +static const _Py_ALIGNED_DEF(64, unsigned char) table_b2a_base85[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" \ "abcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~"; -static const unsigned char table_b2a_base85_a85[] Py_ALIGNED(64) = +static const _Py_ALIGNED_DEF(64, unsigned char) table_b2a_base85_a85[] = "!\"#$%&\'()*+,-./0123456789:;<=>?@" \ "ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstu"; @@ -244,6 +244,129 @@ static const unsigned char table_b2a_base85_a85[] Py_ALIGNED(64) = #define BASE85_A85_Z 0x00000000 #define BASE85_A85_Y 0x20202020 + +static const _Py_ALIGNED_DEF(64, unsigned char) table_a2b_base32[] = { + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,26,27, 28,29,30,31, -1,-1,-1,-1, -1,-1,-1,-1, + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14, + 15,16,17,18, 19,20,21,22, 23,24,25,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, +}; + +static const _Py_ALIGNED_DEF(64, unsigned char) table_b2a_base32[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; + +#define BASE32_PAD '=' + +/* + * Fast base32 encoding/decoding helpers. + * + * Analogous to the helpers for base64. + */ + +/* Encode 5 bytes into 8 base32 characters. */ +static inline void +base32_encode_quint(const unsigned char *in, unsigned char *out, + const unsigned char table[]) +{ + uint64_t combined = ((uint64_t)in[0] << 32) | + ((uint64_t)in[1] << 24) | + ((uint64_t)in[2] << 16) | + ((uint64_t)in[3] << 8) | + (uint64_t)in[4]; + out[0] = table[(combined >> 35) & 0x1f]; + out[1] = table[(combined >> 30) & 0x1f]; + out[2] = table[(combined >> 25) & 0x1f]; + out[3] = table[(combined >> 20) & 0x1f]; + out[4] = table[(combined >> 15) & 0x1f]; + out[5] = table[(combined >> 10) & 0x1f]; + out[6] = table[(combined >> 5) & 0x1f]; + out[7] = table[combined & 0x1f]; +} + +/* + * Encode multiple complete 5-byte groups. + * Returns the number of input bytes processed (always a multiple of 5). + */ +static inline Py_ssize_t +base32_encode_fast(const unsigned char *in, Py_ssize_t in_len, + unsigned char *out, const unsigned char table[]) +{ + Py_ssize_t n_quints = in_len / 5; + const unsigned char *in_end = in + n_quints * 5; + + while (in < in_end) { + base32_encode_quint(in, out, table); + in += 5; + out += 8; + } + + return n_quints * 5; +} + +/* + * Decode 8 base32 characters into 5 bytes. + * Returns 1 on success, 0 if any character is invalid. + */ +static inline int +base32_decode_octa(const unsigned char *in, unsigned char *out, + const unsigned char table[]) +{ + unsigned char v0 = table[in[0]]; + unsigned char v1 = table[in[1]]; + unsigned char v2 = table[in[2]]; + unsigned char v3 = table[in[3]]; + unsigned char v4 = table[in[4]]; + unsigned char v5 = table[in[5]]; + unsigned char v6 = table[in[6]]; + unsigned char v7 = table[in[7]]; + + if ((v0 | v1 | v2 | v3 | v4 | v5 | v6 | v7) & 0xe0) { + return 0; + } + + out[0] = (v0 << 3) | (v1 >> 2); + out[1] = (v1 << 6) | (v2 << 1) | (v3 >> 4); + out[2] = (v3 << 4) | (v4 >> 1); + out[3] = (v4 << 7) | (v5 << 2) | (v6 >> 3); + out[4] = (v6 << 5) | v7; + return 1; +} + +/* + * Decode multiple complete 8-character groups (no padding allowed). + * Returns the number of input characters processed. + * Stops at the first invalid character, padding, or incomplete group. + */ +static inline Py_ssize_t +base32_decode_fast(const unsigned char *in, Py_ssize_t in_len, + unsigned char *out, const unsigned char table[]) +{ + Py_ssize_t n_quints = in_len / 8; + Py_ssize_t i; + + for (i = 0; i < n_quints; i++) { + if (!base32_decode_octa(in + i * 8, out + i * 5, table)) { + break; + } + } + + return i * 8; +} + + static const unsigned short crctab_hqx[256] = { 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, @@ -600,6 +723,8 @@ binascii.a2b_base64 When set to true, bytes that are not part of the base64 standard are not allowed. The same applies to excess data after padding (= / ==). Set to True by default if ignorechars is specified, False otherwise. + padded: bool = True + When set to false, padding in input is not required. alphabet: PyBytesObject(c_default="NULL") = BASE64_ALPHABET ignorechars: Py_buffer = NULL A byte string containing characters to ignore from the input when @@ -610,8 +735,9 @@ Decode a line of base64 data. static PyObject * binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode, - PyBytesObject *alphabet, Py_buffer *ignorechars) -/*[clinic end generated code: output=72f15fcc0681d666 input=195c8d60b03aaa6f]*/ + int padded, PyBytesObject *alphabet, + Py_buffer *ignorechars) +/*[clinic end generated code: output=525d840a299ff132 input=74a53dd3b23474b3]*/ { assert(data->len >= 0); @@ -675,53 +801,51 @@ binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode, /* Check for pad sequences and ignore ** the invalid ones. */ - if (this_ch == BASE64_PAD) { + if (padded && this_ch == BASE64_PAD) { pads++; - - if (strict_mode) { - if (quad_pos >= 2 && quad_pos + pads <= 4) { - continue; - } - if (ignorechar(BASE64_PAD, ignorechars, ignorecache)) { - continue; - } - if (quad_pos == 1) { - /* Set an error below. */ - break; - } - state = get_binascii_state(module); - if (state) { - PyErr_SetString(state->Error, - (quad_pos == 0 && ascii_data == data->buf) - ? "Leading padding not allowed" - : "Excess padding not allowed"); - } - goto error_end; + if (quad_pos >= 2 && quad_pos + pads <= 4) { + continue; } - else { - if (quad_pos >= 2 && quad_pos + pads >= 4) { - /* A pad sequence means we should not parse more input. - ** We've already interpreted the data from the quad at this point. - */ - goto done; - } + // See RFC 4648, section 3.3: "specifications MAY ignore the + // pad character, "=", treating it as non-alphabet data, if + // it is present before the end of the encoded data" and + // "the excess pad characters MAY also be ignored." + if (!strict_mode || ignorechar(BASE64_PAD, ignorechars, ignorecache)) { continue; } + if (quad_pos == 1) { + /* Set an error below. */ + break; + } + state = get_binascii_state(module); + if (state) { + unsigned char *bin_data_start = PyBytesWriter_GetData(writer); + PyErr_SetString(state->Error, + (quad_pos == 0 && bin_data == bin_data_start) + ? "Leading padding not allowed" + : "Excess padding not allowed"); + } + goto error_end; } unsigned char v = table_a2b[this_ch]; if (v >= 64) { + // See RFC 4648, section 3.3. if (strict_mode && !ignorechar(this_ch, ignorechars, ignorecache)) { state = get_binascii_state(module); if (state) { - PyErr_SetString(state->Error, "Only base64 data is allowed"); + PyErr_SetString(state->Error, + (this_ch == BASE64_PAD) + ? "Padding not allowed" + : "Only base64 data is allowed"); } goto error_end; } continue; } - // Characters that are not '=', in the middle of the padding, are not allowed + // Characters that are not '=', in the middle of the padding, are + // not allowed (except when they are). See RFC 4648, section 3.3. if (pads && strict_mode && !ignorechar(BASE64_PAD, ignorechars, ignorecache)) { @@ -777,7 +901,7 @@ binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode, goto error_end; } - if (quad_pos != 0 && quad_pos + pads < 4) { + if (padded && quad_pos != 0 && quad_pos + pads < 4) { state = get_binascii_state(module); if (state) { PyErr_SetString(state->Error, "Incorrect padding"); @@ -785,7 +909,6 @@ binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode, goto error_end; } -done: Py_XDECREF(table_obj); return PyBytesWriter_FinishWithPointer(writer, bin_data); @@ -802,6 +925,8 @@ binascii.b2a_base64 data: Py_buffer / * + padded: bool = True + When set to false, omit padding in the output. wrapcol: size_t = 0 newline: bool = True alphabet: Py_buffer(c_default="{NULL, NULL}") = BASE64_ALPHABET @@ -810,9 +935,9 @@ Base64-code line of data. [clinic start generated code]*/ static PyObject * -binascii_b2a_base64_impl(PyObject *module, Py_buffer *data, size_t wrapcol, - int newline, Py_buffer *alphabet) -/*[clinic end generated code: output=9d9657e5fbe28c64 input=ffa3af8520c312ac]*/ +binascii_b2a_base64_impl(PyObject *module, Py_buffer *data, int padded, + size_t wrapcol, int newline, Py_buffer *alphabet) +/*[clinic end generated code: output=a2057b906dc201ab input=cfa33ad73051d3f7]*/ { const unsigned char *table_b2a = table_b2a_base64; const unsigned char *bin_data = data->buf; @@ -833,6 +958,11 @@ binascii_b2a_base64_impl(PyObject *module, Py_buffer *data, size_t wrapcol, * Use unsigned integer arithmetic to avoid signed integer overflow. */ size_t out_len = ((size_t)bin_len + 2u) / 3u * 4u; + unsigned int pads = (3 - (bin_len % 3)) % 3 * 4 / 3; + if (!padded) { + out_len -= pads; + pads = 0; + } if (wrapcol && out_len) { /* Each line should encode a whole number of bytes. */ wrapcol = wrapcol < 4 ? 4 : wrapcol / 4 * 4; @@ -865,18 +995,23 @@ binascii_b2a_base64_impl(PyObject *module, Py_buffer *data, size_t wrapcol, /* Handle remaining 0-2 bytes */ if (bin_len == 1) { /* 1 byte remaining: produces 2 base64 chars + 2 padding */ + assert(!padded || pads == 2); unsigned int val = bin_data[0]; *ascii_data++ = table_b2a[(val >> 2) & 0x3f]; *ascii_data++ = table_b2a[(val << 4) & 0x3f]; - *ascii_data++ = BASE64_PAD; - *ascii_data++ = BASE64_PAD; } else if (bin_len == 2) { /* 2 bytes remaining: produces 3 base64 chars + 1 padding */ + assert(!padded || pads == 1); unsigned int val = ((unsigned int)bin_data[0] << 8) | bin_data[1]; *ascii_data++ = table_b2a[(val >> 10) & 0x3f]; *ascii_data++ = table_b2a[(val >> 4) & 0x3f]; *ascii_data++ = table_b2a[(val << 2) & 0x3f]; + } + else { + assert(pads == 0); + } + for (; pads; pads--) { *ascii_data++ = BASE64_PAD; } @@ -1178,14 +1313,16 @@ binascii.a2b_base85 / * alphabet: PyBytesObject(c_default="NULL") = BASE85_ALPHABET + ignorechars: Py_buffer = b'' + A byte string containing characters to ignore from the input. Decode a line of Base85 data. [clinic start generated code]*/ static PyObject * binascii_a2b_base85_impl(PyObject *module, Py_buffer *data, - PyBytesObject *alphabet) -/*[clinic end generated code: output=3e114af53812e8ff input=0b6b83b38ad4497c]*/ + PyBytesObject *alphabet, Py_buffer *ignorechars) +/*[clinic end generated code: output=6a8d6eae798818d7 input=04d72a319712bdf3]*/ { const unsigned char *ascii_data = data->buf; Py_ssize_t ascii_len = data->len; @@ -1202,6 +1339,14 @@ binascii_a2b_base85_impl(PyObject *module, Py_buffer *data, table_a2b = (const unsigned char *)PyBytes_AS_STRING(table_obj); } + if (ignorechars->len == 0) { + ignorechars = NULL; + } + ignorecache_t ignorecache; + if (ignorechars != NULL) { + memset(ignorecache, 0, sizeof(ignorecache)); + } + assert(ascii_len >= 0); /* Allocate output buffer. */ @@ -1217,9 +1362,10 @@ binascii_a2b_base85_impl(PyObject *module, Py_buffer *data, int group_pos = 0; for (; ascii_len > 0 || group_pos != 0; ascii_len--, ascii_data++) { /* Shift (in radix-85) data or padding into our buffer. */ + unsigned char this_ch; unsigned char this_digit; if (ascii_len > 0) { - unsigned char this_ch = *ascii_data; + this_ch = *ascii_data; this_digit = table_a2b[this_ch]; } else { @@ -1234,7 +1380,7 @@ binascii_a2b_base85_impl(PyObject *module, Py_buffer *data, state = get_binascii_state(module); if (state != NULL) { PyErr_Format(state->Error, - "Base85 overflow in hunk starting at byte %d", + "Base85 overflow in hunk starting at byte %zd", (data->len - ascii_len) / 5 * 5); } goto error; @@ -1242,10 +1388,10 @@ binascii_a2b_base85_impl(PyObject *module, Py_buffer *data, leftchar = leftchar * 85 + this_digit; group_pos++; } - else { + else if (!ignorechar(this_ch, ignorechars, ignorecache)) { state = get_binascii_state(module); if (state != NULL) { - PyErr_Format(state->Error, "bad Base85 character at position %d", + PyErr_Format(state->Error, "bad Base85 character at position %zd", data->len - ascii_len); } goto error; @@ -1283,6 +1429,7 @@ binascii.b2a_base85 * pad: bool = False Pad input to a multiple of 4 before encoding. + wrapcol: size_t = 0 alphabet: Py_buffer(c_default="{NULL, NULL}") = BASE85_ALPHABET Base85-code line of data. @@ -1290,8 +1437,8 @@ Base85-code line of data. static PyObject * binascii_b2a_base85_impl(PyObject *module, Py_buffer *data, int pad, - Py_buffer *alphabet) -/*[clinic end generated code: output=a59f4f2ff6f0e69f input=30f545c6ff554db7]*/ + size_t wrapcol, Py_buffer *alphabet) +/*[clinic end generated code: output=98b962ed52c776a4 input=1b20b0bd6572691b]*/ { const unsigned char *bin_data = data->buf; Py_ssize_t bin_len = data->len; @@ -1312,6 +1459,11 @@ binascii_b2a_base85_impl(PyObject *module, Py_buffer *data, int pad, if (!pad && (bin_len % 4)) { out_len -= 4 - (bin_len % 4); } + if (wrapcol && out_len) { + /* Each line should encode a whole number of bytes. */ + wrapcol = wrapcol < 5 ? 5 : wrapcol / 5 * 5; + out_len += (out_len - 1u) / wrapcol; + } if (out_len > PY_SSIZE_T_MAX) { binascii_state *state = get_binascii_state(module); if (state == NULL) { @@ -1364,6 +1516,347 @@ binascii_b2a_base85_impl(PyObject *module, Py_buffer *data, int pad, ascii_data += group_len; } + if (wrapcol && out_len) { + unsigned char *start = PyBytesWriter_GetData(writer); + ascii_data = start + wraplines(start, ascii_data - start, wrapcol); + } + + return PyBytesWriter_FinishWithPointer(writer, ascii_data); +} + +/*[clinic input] +binascii.a2b_base32 + + data: ascii_buffer + / + * + padded: bool = True + When set to false, padding in input is not required. + alphabet: PyBytesObject(c_default="NULL") = BASE32_ALPHABET + ignorechars: Py_buffer = b'' + A byte string containing characters to ignore from the input. + +Decode a line of base32 data. +[clinic start generated code]*/ + +static PyObject * +binascii_a2b_base32_impl(PyObject *module, Py_buffer *data, int padded, + PyBytesObject *alphabet, Py_buffer *ignorechars) +/*[clinic end generated code: output=7dbbaa816d956b1c input=07a3721acdf9b688]*/ +{ + const unsigned char *ascii_data = data->buf; + Py_ssize_t ascii_len = data->len; + binascii_state *state = NULL; + PyObject *table_obj = NULL; + const unsigned char *table_a2b = table_a2b_base32; + + assert(ascii_len >= 0); + + if (alphabet != NULL) { + state = get_binascii_state(module); + table_obj = get_reverse_table(state, (PyObject *)alphabet, 32, BASE32_PAD); + if (table_obj == NULL) { + return NULL; + } + table_a2b = (const unsigned char *)PyBytes_AS_STRING(table_obj); + } + + if (ignorechars->len == 0) { + ignorechars = NULL; + } + ignorecache_t ignorecache; + if (ignorechars != NULL) { + memset(ignorecache, 0, sizeof(ignorecache)); + } + + /* Allocate output buffer. */ + size_t bin_len = ((size_t)ascii_len + 7) / 8 * 5; + PyBytesWriter *writer = PyBytesWriter_Create(bin_len); + if (writer == NULL) { + Py_XDECREF(table_obj); + return NULL; + } + unsigned char *bin_data = PyBytesWriter_GetData(writer); + +fastpath: + /* + * Fast path: use optimized decoder for complete octas (groups of 8 bytes). + * The fast path stops at padding, invalid chars, or incomplete octas. + */ + if (ascii_len >= 8) { + Py_ssize_t fast_chars = base32_decode_fast(ascii_data, ascii_len, + bin_data, table_a2b); + if (fast_chars > 0) { + ascii_data += fast_chars; + ascii_len -= fast_chars; + bin_data += (fast_chars / 8) * 5; + } + } + + /* Slow path: handle remaining input (padding, invalid chars, incomplete octas). */ + unsigned char leftchar = 0; + int octa_pos = 0; + int pads = 0; + for (; ascii_len; ascii_len--, ascii_data++) { + unsigned char this_ch = *ascii_data; + + /* Check for pad sequences. They may only occur at certain positions. */ + if (padded && this_ch == BASE32_PAD) { + pads++; + + if ((octa_pos == 2 || octa_pos == 4 || octa_pos == 5 || octa_pos == 7) + && octa_pos + pads <= 8) + { + continue; + } + // See RFC 4648, section 3.3: "specifications MAY ignore the + // pad character, "=", treating it as non-alphabet data, if + // it is present before the end of the encoded data" and + // "the excess pad characters MAY also be ignored." + if (ignorechar(BASE32_PAD, ignorechars, ignorecache)) { + continue; + } + if (octa_pos == 1 || octa_pos == 3 || octa_pos == 6) { + /* Set an error below. */ + break; + } + state = get_binascii_state(module); + if (state) { + unsigned char *bin_data_start = PyBytesWriter_GetData(writer); + PyErr_SetString(state->Error, + (octa_pos == 0 && bin_data == bin_data_start) + ? "Leading padding not allowed" + : "Excess padding not allowed"); + } + goto error; + } + + unsigned char v = table_a2b[this_ch]; + if (v >= 32) { + // See RFC 4648, section 3.3. + if (!ignorechar(this_ch, ignorechars, ignorecache)) { + state = get_binascii_state(module); + if (state) { + PyErr_SetString(state->Error, + (this_ch == BASE32_PAD) + ? "Padding not allowed" + : "Only base32 data is allowed"); + } + goto error; + } + continue; + } + + // Characters that are not '=', in the middle of the padding, are + // not allowed (except when they are). See RFC 4648, section 3.3. + if (pads && !ignorechar(BASE32_PAD, ignorechars, ignorecache)) { + state = get_binascii_state(module); + if (state) { + PyErr_SetString(state->Error, (octa_pos + pads == 8) + ? "Excess data after padding" + : "Discontinuous padding not allowed"); + } + goto error; + } + + switch (octa_pos) { + case 0: + octa_pos = 1; + leftchar = v; + break; + case 1: + octa_pos = 2; + *bin_data++ = (leftchar << 3) | (v >> 2); + leftchar = v & 0x03; + break; + case 2: + octa_pos = 3; + leftchar = (leftchar << 5) | v; + break; + case 3: + octa_pos = 4; + *bin_data++ = (leftchar << 1) | (v >> 4); + leftchar = v & 0x0f; + break; + case 4: + octa_pos = 5; + *bin_data++ = (leftchar << 4) | (v >> 1); + leftchar = v & 0x01; + break; + case 5: + octa_pos = 6; + leftchar = (leftchar << 5) | v; + break; + case 6: + octa_pos = 7; + *bin_data++ = (leftchar << 2) | (v >> 3); + leftchar = v & 0x07; + break; + case 7: + octa_pos = 0; + *bin_data++ = (leftchar << 5) | v; + leftchar = 0; + ascii_data++; + ascii_len--; + goto fastpath; + } + } + + if (octa_pos == 1 || octa_pos == 3 || octa_pos == 6) { + state = get_binascii_state(module); + if (state) { + unsigned char *bin_data_start = PyBytesWriter_GetData(writer); + PyErr_Format(state->Error, + "Invalid base32-encoded string: " + "number of data characters (%zd) " + "cannot be 1, 3, or 6 more than a multiple of 8", + (bin_data - bin_data_start) / 5 * 8 + octa_pos); + } + goto error; + } + + if (padded && octa_pos != 0 && octa_pos + pads < 8) { + state = get_binascii_state(module); + if (state) { + PyErr_SetString(state->Error, "Incorrect padding"); + } + goto error; + } + + Py_XDECREF(table_obj); + return PyBytesWriter_FinishWithPointer(writer, bin_data); + +error: + PyBytesWriter_Discard(writer); + Py_XDECREF(table_obj); + return NULL; +} + +/*[clinic input] +binascii.b2a_base32 + + data: Py_buffer + / + * + padded: bool = True + When set to false, omit padding in the output. + wrapcol: size_t = 0 + alphabet: Py_buffer(c_default="{NULL, NULL}") = BASE32_ALPHABET + +Base32-code line of data. +[clinic start generated code]*/ + +static PyObject * +binascii_b2a_base32_impl(PyObject *module, Py_buffer *data, int padded, + size_t wrapcol, Py_buffer *alphabet) +/*[clinic end generated code: output=acc09e685569aab9 input=1889b0c497a1d3c2]*/ +{ + const unsigned char *table_b2a = table_b2a_base32; + const unsigned char *bin_data = data->buf; + Py_ssize_t bin_len = data->len; + binascii_state *state = NULL; + + assert(bin_len >= 0); + + if (alphabet->buf != NULL) { + if (alphabet->len != 32) { + PyErr_SetString(PyExc_ValueError, "alphabet must have length 32"); + return NULL; + } + table_b2a = alphabet->buf; + } + + /* + * Each group of 5 bytes (rounded up) gets encoded as 8 characters. + * Use unsigned integer arithmetic to avoid signed integer overflow. + */ + size_t ascii_len = ((size_t)bin_len + 4u) / 5u * 8u; + unsigned int pads = (5 - (bin_len % 5)) % 5 * 8 / 5; + if (!padded) { + ascii_len -= pads; + pads = 0; + } + if (wrapcol && ascii_len) { + /* Each line should encode a whole number of bytes. */ + wrapcol = wrapcol < 8 ? 8 : wrapcol / 8 * 8; + ascii_len += (ascii_len - 1u) / wrapcol; + } + if (ascii_len > PY_SSIZE_T_MAX) { + state = get_binascii_state(module); + if (state) { + PyErr_SetString(state->Error, "Too much data for base32"); + } + return NULL; + } + PyBytesWriter *writer = PyBytesWriter_Create(ascii_len); + if (writer == NULL) { + return NULL; + } + unsigned char *ascii_data = PyBytesWriter_GetData(writer); + + /* Use the optimized fast path for complete 5-byte groups. */ + Py_ssize_t fast_bytes = base32_encode_fast(bin_data, bin_len, ascii_data, + table_b2a); + bin_data += fast_bytes; + ascii_data += (fast_bytes / 5) * 8; + bin_len -= fast_bytes; + + /* Handle the remaining 0-4 bytes. */ + if (bin_len == 1) { + /* 1 byte remaining: produces 2 encoded + 6 padding chars. */ + assert(!padded || pads == 6); + uint32_t val = bin_data[0]; + *ascii_data++ = table_b2a[(val >> 3) & 0x1f]; + *ascii_data++ = table_b2a[(val << 2) & 0x1f]; + } + else if (bin_len == 2) { + /* 2 bytes remaining: produces 4 encoded + 4 padding chars. */ + assert(!padded || pads == 4); + uint32_t val = ((uint32_t)bin_data[0] << 8) | bin_data[1]; + *ascii_data++ = table_b2a[(val >> 11) & 0x1f]; + *ascii_data++ = table_b2a[(val >> 6) & 0x1f]; + *ascii_data++ = table_b2a[(val >> 1) & 0x1f]; + *ascii_data++ = table_b2a[(val << 4) & 0x1f]; + } + else if (bin_len == 3) { + /* 3 bytes remaining: produces 5 encoded + 3 padding chars. */ + assert(!padded || pads == 3); + uint32_t val = ((uint32_t)bin_data[0] << 16) + | ((uint32_t)bin_data[1] << 8) + | bin_data[2]; + *ascii_data++ = table_b2a[(val >> 19) & 0x1f]; + *ascii_data++ = table_b2a[(val >> 14) & 0x1f]; + *ascii_data++ = table_b2a[(val >> 9) & 0x1f]; + *ascii_data++ = table_b2a[(val >> 4) & 0x1f]; + *ascii_data++ = table_b2a[(val << 1) & 0x1f]; + } + else if (bin_len == 4) { + /* 4 bytes remaining: produces 7 encoded + 1 padding chars. */ + assert(!padded || pads == 1); + uint32_t val = ((uint32_t)bin_data[0] << 24) + | ((uint32_t)bin_data[1] << 16) + | ((uint32_t)bin_data[2] << 8) + | bin_data[3]; + *ascii_data++ = table_b2a[(val >> 27) & 0x1f]; + *ascii_data++ = table_b2a[(val >> 22) & 0x1f]; + *ascii_data++ = table_b2a[(val >> 17) & 0x1f]; + *ascii_data++ = table_b2a[(val >> 12) & 0x1f]; + *ascii_data++ = table_b2a[(val >> 7) & 0x1f]; + *ascii_data++ = table_b2a[(val >> 2) & 0x1f]; + *ascii_data++ = table_b2a[(val << 3) & 0x1f]; + } + else { + assert(pads == 0); + } + for (; pads; pads--) { + *ascii_data++ = BASE32_PAD; + } + + if (wrapcol && ascii_len) { + unsigned char *start = PyBytesWriter_GetData(writer); + ascii_data = start + wraplines(start, ascii_data - start, wrapcol); + } + return PyBytesWriter_FinishWithPointer(writer, ascii_data); } @@ -1602,7 +2095,7 @@ binascii.b2a_hex data: Py_buffer sep: object = NULL An optional single character or byte to separate hex bytes. - bytes_per_sep: int = 1 + bytes_per_sep: Py_ssize_t = 1 How many bytes between separators. Positive values count from the right, negative values count from the left. @@ -1622,8 +2115,8 @@ b'b9_01ef' static PyObject * binascii_b2a_hex_impl(PyObject *module, Py_buffer *data, PyObject *sep, - int bytes_per_sep) -/*[clinic end generated code: output=a26937946a81d2c7 input=ec0ade6ba2e43543]*/ + Py_ssize_t bytes_per_sep) +/*[clinic end generated code: output=7d703f866f74a813 input=6a1606f01a87118c]*/ { return _Py_strhex_bytes_with_sep((const char *)data->buf, data->len, sep, bytes_per_sep); @@ -1640,8 +2133,8 @@ available as "b2a_hex()". static PyObject * binascii_hexlify_impl(PyObject *module, Py_buffer *data, PyObject *sep, - int bytes_per_sep) -/*[clinic end generated code: output=d12aa1b001b15199 input=bc317bd4e241f76b]*/ + Py_ssize_t bytes_per_sep) +/*[clinic end generated code: output=b99b3b39d234a3d4 input=bc317bd4e241f76b]*/ { return _Py_strhex_bytes_with_sep((const char *)data->buf, data->len, sep, bytes_per_sep); @@ -1652,6 +2145,9 @@ binascii.a2b_hex hexstr: ascii_buffer / + * + ignorechars: Py_buffer = b'' + A byte string containing characters to ignore from the input. Binary data of hexadecimal representation. @@ -1660,53 +2156,68 @@ This function is also available as "unhexlify()". [clinic start generated code]*/ static PyObject * -binascii_a2b_hex_impl(PyObject *module, Py_buffer *hexstr) -/*[clinic end generated code: output=0cc1a139af0eeecb input=9e1e7f2f94db24fd]*/ +binascii_a2b_hex_impl(PyObject *module, Py_buffer *hexstr, + Py_buffer *ignorechars) +/*[clinic end generated code: output=021a7ed5a742cb20 input=6154b3f4e6e2c0c3]*/ { - const char* argbuf; - Py_ssize_t arglen; - Py_ssize_t i, j; - binascii_state *state; - - argbuf = hexstr->buf; - arglen = hexstr->len; - - assert(arglen >= 0); + const unsigned char *ascii_data = hexstr->buf; + size_t ascii_len = hexstr->len; + binascii_state *state = NULL; - /* XXX What should we do about strings with an odd length? Should - * we add an implicit leading zero, or a trailing zero? For now, - * raise an exception. - */ - if (arglen % 2) { - state = get_binascii_state(module); - if (state == NULL) { - return NULL; - } - PyErr_SetString(state->Error, "Odd-length string"); - return NULL; + if (ignorechars->len == 0) { + ignorechars = NULL; + } + ignorecache_t ignorecache; + if (ignorechars != NULL) { + memset(ignorecache, 0, sizeof(ignorecache)); } - PyBytesWriter *writer = PyBytesWriter_Create(arglen/2); + /* Allocate the buffer */ + Py_ssize_t bin_len = ascii_len/2; + PyBytesWriter *writer = PyBytesWriter_Create(bin_len); if (writer == NULL) { return NULL; } - char *retbuf = PyBytesWriter_GetData(writer); + unsigned char *bin_data = PyBytesWriter_GetData(writer); - for (i=j=0; i < arglen; i += 2) { - unsigned int top = _PyLong_DigitValue[Py_CHARMASK(argbuf[i])]; - unsigned int bot = _PyLong_DigitValue[Py_CHARMASK(argbuf[i+1])]; - if (top >= 16 || bot >= 16) { - state = get_binascii_state(module); - if (state == NULL) { + int pair_pos = 0; + unsigned char leftchar = 0; + for (; ascii_len; ascii_data++, ascii_len--) { + unsigned char this_ch = *ascii_data; + + unsigned char this_digit = _PyLong_DigitValue[this_ch]; + if (this_digit >= 16) { + // See RFC 4648, section 3.3. + if (!ignorechar(this_ch, ignorechars, ignorecache)) { + state = get_binascii_state(module); + if (state) { + PyErr_SetString(state->Error, + "Non-hexadecimal digit found"); + } goto error; } - PyErr_SetString(state->Error, - "Non-hexadecimal digit found"); - goto error; + continue; + } + + if (!pair_pos) { + pair_pos = 1; + leftchar = this_digit; + } + else { + pair_pos = 0; + *bin_data++ = (leftchar << 4) | this_digit; } - retbuf[j++] = (top << 4) + bot; } - return PyBytesWriter_Finish(writer); + + if (pair_pos) { + state = get_binascii_state(module); + if (state) { + PyErr_SetString(state->Error, "Odd number of hexadecimal digits"); + } + goto error; + } + + return PyBytesWriter_FinishWithPointer(writer, bin_data); error: PyBytesWriter_Discard(writer); @@ -1722,10 +2233,11 @@ hexstr must contain an even number of hex digits (upper or lower case). [clinic start generated code]*/ static PyObject * -binascii_unhexlify_impl(PyObject *module, Py_buffer *hexstr) -/*[clinic end generated code: output=51a64c06c79629e3 input=dd8c012725f462da]*/ +binascii_unhexlify_impl(PyObject *module, Py_buffer *hexstr, + Py_buffer *ignorechars) +/*[clinic end generated code: output=40e87f8a0ded5880 input=dd8c012725f462da]*/ { - return binascii_a2b_hex_impl(module, hexstr); + return binascii_a2b_hex_impl(module, hexstr, ignorechars); } #define MAXLINESIZE 76 @@ -2028,6 +2540,8 @@ static struct PyMethodDef binascii_module_methods[] = { BINASCII_A2B_ASCII85_METHODDEF BINASCII_A2B_BASE85_METHODDEF BINASCII_B2A_BASE85_METHODDEF + BINASCII_A2B_BASE32_METHODDEF + BINASCII_B2A_BASE32_METHODDEF BINASCII_A2B_HEX_METHODDEF BINASCII_B2A_HEX_METHODDEF BINASCII_HEXLIFY_METHODDEF @@ -2114,9 +2628,14 @@ binascii_exec(PyObject *module) { return -1; } - - state->reverse_table_cache = PyDict_New(); - if (state->reverse_table_cache == NULL) { + if (PyModule_Add(module, "BASE32_ALPHABET", + PyBytes_FromStringAndSize((const char *)table_b2a_base32, 32)) < 0) + { + return -1; + } + if (PyModule_Add(module, "BASE32HEX_ALPHABET", + PyBytes_FromString("0123456789ABCDEFGHIJKLMNOPQRSTUV")) < 0) + { return -1; } @@ -2129,6 +2648,7 @@ binascii_exec(PyObject *module) } static PyModuleDef_Slot binascii_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, binascii_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/blake2module.c b/Modules/blake2module.c index bb3b934be69dd7..b71dd20925611e 100644 --- a/Modules/blake2module.c +++ b/Modules/blake2module.c @@ -280,6 +280,7 @@ blake2_exec(PyObject *m) } static PyModuleDef_Slot _blake2_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, blake2_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/cjkcodecs/cjkcodecs.h b/Modules/cjkcodecs/cjkcodecs.h index f66412237011d4..9d86396f73b2b5 100644 --- a/Modules/cjkcodecs/cjkcodecs.h +++ b/Modules/cjkcodecs/cjkcodecs.h @@ -500,6 +500,7 @@ static struct PyMethodDef _cjk_methods[] = { }; static PyModuleDef_Slot _cjk_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _cjk_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index a7fac2380f2519..f1124147e2b0a7 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -12,6 +12,7 @@ #include "multibytecodec.h" #include "clinic/multibytecodec.c.h" +#include "pycore_tuple.h" // _PyTuple_FromPairSteal #include // offsetof() @@ -102,26 +103,17 @@ static PyObject *multibytecodec_encode(const MultibyteCodec *, static PyObject * make_tuple(PyObject *object, Py_ssize_t len) { - PyObject *v, *w; - - if (object == NULL) - return NULL; - - v = PyTuple_New(2); - if (v == NULL) { - Py_DECREF(object); + if (object == NULL) { return NULL; } - PyTuple_SET_ITEM(v, 0, object); - w = PyLong_FromSsize_t(len); - if (w == NULL) { - Py_DECREF(v); + PyObject* len_obj = PyLong_FromSsize_t(len); + if (len_obj == NULL) { + Py_DECREF(object); return NULL; } - PyTuple_SET_ITEM(v, 1, w); - return v; + return _PyTuple_FromPairSteal(object, len_obj); } static PyObject * @@ -2121,6 +2113,7 @@ static struct PyMethodDef _multibytecodec_methods[] = { }; static PyModuleDef_Slot _multibytecodec_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _multibytecodec_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index 2648583c654a04..8a3fb4b515e4ac 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -335,8 +335,9 @@ PyDoc_STRVAR(array_array_byteswap__doc__, "\n" "Byteswap all items of the array.\n" "\n" -"If the items in the array are not 1, 2, 4, or 8 bytes in size, RuntimeError is\n" -"raised."); +"If the items in the array are not 1, 2, 4, 8 or 16 bytes in size, RuntimeError\n" +"is raised. Note, that for complex types the order of\n" +"components (the real part, followed by imaginary part) is preserved."); #define ARRAY_ARRAY_BYTESWAP_METHODDEF \ {"byteswap", (PyCFunction)array_array_byteswap, METH_NOARGS, array_array_byteswap__doc__}, @@ -778,4 +779,4 @@ array_arrayiterator___setstate__(PyObject *self, PyObject *state) return return_value; } -/*[clinic end generated code: output=c993c3598085840e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9dcb2fc40710f83d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h index 2fdecc2efbf9d4..0a2d33c428d10a 100644 --- a/Modules/clinic/binascii.c.h +++ b/Modules/clinic/binascii.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_long.h" // _PyLong_Size_t_Converter() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() @@ -117,7 +118,8 @@ binascii_b2a_uu(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyDoc_STRVAR(binascii_a2b_base64__doc__, "a2b_base64($module, data, /, *, strict_mode=,\n" -" alphabet=BASE64_ALPHABET, ignorechars=)\n" +" padded=True, alphabet=BASE64_ALPHABET,\n" +" ignorechars=)\n" "--\n" "\n" "Decode a line of base64 data.\n" @@ -126,6 +128,8 @@ PyDoc_STRVAR(binascii_a2b_base64__doc__, " When set to true, bytes that are not part of the base64 standard are\n" " not allowed. The same applies to excess data after padding (= / ==).\n" " Set to True by default if ignorechars is specified, False otherwise.\n" +" padded\n" +" When set to false, padding in input is not required.\n" " ignorechars\n" " A byte string containing characters to ignore from the input when\n" " strict_mode is true."); @@ -135,7 +139,8 @@ PyDoc_STRVAR(binascii_a2b_base64__doc__, static PyObject * binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode, - PyBytesObject *alphabet, Py_buffer *ignorechars); + int padded, PyBytesObject *alphabet, + Py_buffer *ignorechars); static PyObject * binascii_a2b_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -143,7 +148,7 @@ binascii_a2b_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 3 + #define NUM_KEYWORDS 4 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -152,7 +157,7 @@ binascii_a2b_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(strict_mode), &_Py_ID(alphabet), &_Py_ID(ignorechars), }, + .ob_item = { &_Py_ID(strict_mode), &_Py_ID(padded), &_Py_ID(alphabet), &_Py_ID(ignorechars), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -161,17 +166,18 @@ binascii_a2b_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"", "strict_mode", "alphabet", "ignorechars", NULL}; + static const char * const _keywords[] = {"", "strict_mode", "padded", "alphabet", "ignorechars", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "a2b_base64", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[4]; + PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; Py_buffer data = {NULL, NULL}; int strict_mode = -1; + int padded = 1; PyBytesObject *alphabet = NULL; Py_buffer ignorechars = {NULL, NULL}; @@ -196,20 +202,29 @@ binascii_a2b_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P } } if (args[2]) { - if (!PyBytes_Check(args[2])) { - _PyArg_BadArgument("a2b_base64", "argument 'alphabet'", "bytes", args[2]); + padded = PyObject_IsTrue(args[2]); + if (padded < 0) { goto exit; } - alphabet = (PyBytesObject *)args[2]; if (!--noptargs) { goto skip_optional_kwonly; } } - if (PyObject_GetBuffer(args[3], &ignorechars, PyBUF_SIMPLE) != 0) { + if (args[3]) { + if (!PyBytes_Check(args[3])) { + _PyArg_BadArgument("a2b_base64", "argument 'alphabet'", "bytes", args[3]); + goto exit; + } + alphabet = (PyBytesObject *)args[3]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (PyObject_GetBuffer(args[4], &ignorechars, PyBUF_SIMPLE) != 0) { goto exit; } skip_optional_kwonly: - return_value = binascii_a2b_base64_impl(module, &data, strict_mode, alphabet, &ignorechars); + return_value = binascii_a2b_base64_impl(module, &data, strict_mode, padded, alphabet, &ignorechars); exit: /* Cleanup for data */ @@ -224,18 +239,21 @@ binascii_a2b_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P } PyDoc_STRVAR(binascii_b2a_base64__doc__, -"b2a_base64($module, data, /, *, wrapcol=0, newline=True,\n" +"b2a_base64($module, data, /, *, padded=True, wrapcol=0, newline=True,\n" " alphabet=BASE64_ALPHABET)\n" "--\n" "\n" -"Base64-code line of data."); +"Base64-code line of data.\n" +"\n" +" padded\n" +" When set to false, omit padding in the output."); #define BINASCII_B2A_BASE64_METHODDEF \ {"b2a_base64", _PyCFunction_CAST(binascii_b2a_base64), METH_FASTCALL|METH_KEYWORDS, binascii_b2a_base64__doc__}, static PyObject * -binascii_b2a_base64_impl(PyObject *module, Py_buffer *data, size_t wrapcol, - int newline, Py_buffer *alphabet); +binascii_b2a_base64_impl(PyObject *module, Py_buffer *data, int padded, + size_t wrapcol, int newline, Py_buffer *alphabet); static PyObject * binascii_b2a_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -243,7 +261,7 @@ binascii_b2a_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 3 + #define NUM_KEYWORDS 4 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -252,7 +270,7 @@ binascii_b2a_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(wrapcol), &_Py_ID(newline), &_Py_ID(alphabet), }, + .ob_item = { &_Py_ID(padded), &_Py_ID(wrapcol), &_Py_ID(newline), &_Py_ID(alphabet), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -261,16 +279,17 @@ binascii_b2a_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"", "wrapcol", "newline", "alphabet", NULL}; + static const char * const _keywords[] = {"", "padded", "wrapcol", "newline", "alphabet", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "b2a_base64", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[4]; + PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; Py_buffer data = {NULL, NULL}; + int padded = 1; size_t wrapcol = 0; int newline = 1; Py_buffer alphabet = {NULL, NULL}; @@ -287,7 +306,8 @@ binascii_b2a_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P goto skip_optional_kwonly; } if (args[1]) { - if (!_PyLong_Size_t_Converter(args[1], &wrapcol)) { + padded = PyObject_IsTrue(args[1]); + if (padded < 0) { goto exit; } if (!--noptargs) { @@ -295,7 +315,15 @@ binascii_b2a_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P } } if (args[2]) { - newline = PyObject_IsTrue(args[2]); + if (!_PyLong_Size_t_Converter(args[2], &wrapcol)) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (args[3]) { + newline = PyObject_IsTrue(args[3]); if (newline < 0) { goto exit; } @@ -303,11 +331,11 @@ binascii_b2a_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P goto skip_optional_kwonly; } } - if (PyObject_GetBuffer(args[3], &alphabet, PyBUF_SIMPLE) != 0) { + if (PyObject_GetBuffer(args[4], &alphabet, PyBUF_SIMPLE) != 0) { goto exit; } skip_optional_kwonly: - return_value = binascii_b2a_base64_impl(module, &data, wrapcol, newline, &alphabet); + return_value = binascii_b2a_base64_impl(module, &data, padded, wrapcol, newline, &alphabet); exit: /* Cleanup for data */ @@ -544,17 +572,21 @@ binascii_b2a_ascii85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, } PyDoc_STRVAR(binascii_a2b_base85__doc__, -"a2b_base85($module, data, /, *, alphabet=BASE85_ALPHABET)\n" +"a2b_base85($module, data, /, *, alphabet=BASE85_ALPHABET,\n" +" ignorechars=b\'\')\n" "--\n" "\n" -"Decode a line of Base85 data."); +"Decode a line of Base85 data.\n" +"\n" +" ignorechars\n" +" A byte string containing characters to ignore from the input."); #define BINASCII_A2B_BASE85_METHODDEF \ {"a2b_base85", _PyCFunction_CAST(binascii_a2b_base85), METH_FASTCALL|METH_KEYWORDS, binascii_a2b_base85__doc__}, static PyObject * binascii_a2b_base85_impl(PyObject *module, Py_buffer *data, - PyBytesObject *alphabet); + PyBytesObject *alphabet, Py_buffer *ignorechars); static PyObject * binascii_a2b_base85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -562,7 +594,7 @@ binascii_a2b_base85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 2 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -571,7 +603,7 @@ binascii_a2b_base85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(alphabet), }, + .ob_item = { &_Py_ID(alphabet), &_Py_ID(ignorechars), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -580,17 +612,18 @@ binascii_a2b_base85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"", "alphabet", NULL}; + static const char * const _keywords[] = {"", "alphabet", "ignorechars", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "a2b_base85", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; Py_buffer data = {NULL, NULL}; PyBytesObject *alphabet = NULL; + Py_buffer ignorechars = {.buf = "", .obj = NULL, .len = 0}; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -603,24 +636,37 @@ binascii_a2b_base85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (!noptargs) { goto skip_optional_kwonly; } - if (!PyBytes_Check(args[1])) { - _PyArg_BadArgument("a2b_base85", "argument 'alphabet'", "bytes", args[1]); + if (args[1]) { + if (!PyBytes_Check(args[1])) { + _PyArg_BadArgument("a2b_base85", "argument 'alphabet'", "bytes", args[1]); + goto exit; + } + alphabet = (PyBytesObject *)args[1]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (PyObject_GetBuffer(args[2], &ignorechars, PyBUF_SIMPLE) != 0) { goto exit; } - alphabet = (PyBytesObject *)args[1]; skip_optional_kwonly: - return_value = binascii_a2b_base85_impl(module, &data, alphabet); + return_value = binascii_a2b_base85_impl(module, &data, alphabet, &ignorechars); exit: /* Cleanup for data */ if (data.obj) PyBuffer_Release(&data); + /* Cleanup for ignorechars */ + if (ignorechars.obj) { + PyBuffer_Release(&ignorechars); + } return return_value; } PyDoc_STRVAR(binascii_b2a_base85__doc__, -"b2a_base85($module, data, /, *, pad=False, alphabet=BASE85_ALPHABET)\n" +"b2a_base85($module, data, /, *, pad=False, wrapcol=0,\n" +" alphabet=BASE85_ALPHABET)\n" "--\n" "\n" "Base85-code line of data.\n" @@ -633,7 +679,7 @@ PyDoc_STRVAR(binascii_b2a_base85__doc__, static PyObject * binascii_b2a_base85_impl(PyObject *module, Py_buffer *data, int pad, - Py_buffer *alphabet); + size_t wrapcol, Py_buffer *alphabet); static PyObject * binascii_b2a_base85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -641,7 +687,7 @@ binascii_b2a_base85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -650,7 +696,7 @@ binascii_b2a_base85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(pad), &_Py_ID(alphabet), }, + .ob_item = { &_Py_ID(pad), &_Py_ID(wrapcol), &_Py_ID(alphabet), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -659,17 +705,18 @@ binascii_b2a_base85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"", "pad", "alphabet", NULL}; + static const char * const _keywords[] = {"", "pad", "wrapcol", "alphabet", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "b2a_base85", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; Py_buffer data = {NULL, NULL}; int pad = 0; + size_t wrapcol = 0; Py_buffer alphabet = {NULL, NULL}; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, @@ -692,11 +739,226 @@ binascii_b2a_base85(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P goto skip_optional_kwonly; } } - if (PyObject_GetBuffer(args[2], &alphabet, PyBUF_SIMPLE) != 0) { + if (args[2]) { + if (!_PyLong_Size_t_Converter(args[2], &wrapcol)) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (PyObject_GetBuffer(args[3], &alphabet, PyBUF_SIMPLE) != 0) { + goto exit; + } +skip_optional_kwonly: + return_value = binascii_b2a_base85_impl(module, &data, pad, wrapcol, &alphabet); + +exit: + /* Cleanup for data */ + if (data.obj) { + PyBuffer_Release(&data); + } + /* Cleanup for alphabet */ + if (alphabet.obj) { + PyBuffer_Release(&alphabet); + } + + return return_value; +} + +PyDoc_STRVAR(binascii_a2b_base32__doc__, +"a2b_base32($module, data, /, *, padded=True, alphabet=BASE32_ALPHABET,\n" +" ignorechars=b\'\')\n" +"--\n" +"\n" +"Decode a line of base32 data.\n" +"\n" +" padded\n" +" When set to false, padding in input is not required.\n" +" ignorechars\n" +" A byte string containing characters to ignore from the input."); + +#define BINASCII_A2B_BASE32_METHODDEF \ + {"a2b_base32", _PyCFunction_CAST(binascii_a2b_base32), METH_FASTCALL|METH_KEYWORDS, binascii_a2b_base32__doc__}, + +static PyObject * +binascii_a2b_base32_impl(PyObject *module, Py_buffer *data, int padded, + PyBytesObject *alphabet, Py_buffer *ignorechars); + +static PyObject * +binascii_a2b_base32(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(padded), &_Py_ID(alphabet), &_Py_ID(ignorechars), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "padded", "alphabet", "ignorechars", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "a2b_base32", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + Py_buffer data = {NULL, NULL}; + int padded = 1; + PyBytesObject *alphabet = NULL; + Py_buffer ignorechars = {.buf = "", .obj = NULL, .len = 0}; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!ascii_buffer_converter(args[0], &data)) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[1]) { + padded = PyObject_IsTrue(args[1]); + if (padded < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (args[2]) { + if (!PyBytes_Check(args[2])) { + _PyArg_BadArgument("a2b_base32", "argument 'alphabet'", "bytes", args[2]); + goto exit; + } + alphabet = (PyBytesObject *)args[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (PyObject_GetBuffer(args[3], &ignorechars, PyBUF_SIMPLE) != 0) { + goto exit; + } +skip_optional_kwonly: + return_value = binascii_a2b_base32_impl(module, &data, padded, alphabet, &ignorechars); + +exit: + /* Cleanup for data */ + if (data.obj) + PyBuffer_Release(&data); + /* Cleanup for ignorechars */ + if (ignorechars.obj) { + PyBuffer_Release(&ignorechars); + } + + return return_value; +} + +PyDoc_STRVAR(binascii_b2a_base32__doc__, +"b2a_base32($module, data, /, *, padded=True, wrapcol=0,\n" +" alphabet=BASE32_ALPHABET)\n" +"--\n" +"\n" +"Base32-code line of data.\n" +"\n" +" padded\n" +" When set to false, omit padding in the output."); + +#define BINASCII_B2A_BASE32_METHODDEF \ + {"b2a_base32", _PyCFunction_CAST(binascii_b2a_base32), METH_FASTCALL|METH_KEYWORDS, binascii_b2a_base32__doc__}, + +static PyObject * +binascii_b2a_base32_impl(PyObject *module, Py_buffer *data, int padded, + size_t wrapcol, Py_buffer *alphabet); + +static PyObject * +binascii_b2a_base32(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(padded), &_Py_ID(wrapcol), &_Py_ID(alphabet), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "padded", "wrapcol", "alphabet", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "b2a_base32", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + Py_buffer data = {NULL, NULL}; + int padded = 1; + size_t wrapcol = 0; + Py_buffer alphabet = {NULL, NULL}; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[1]) { + padded = PyObject_IsTrue(args[1]); + if (padded < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (args[2]) { + if (!_PyLong_Size_t_Converter(args[2], &wrapcol)) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (PyObject_GetBuffer(args[3], &alphabet, PyBUF_SIMPLE) != 0) { goto exit; } skip_optional_kwonly: - return_value = binascii_b2a_base85_impl(module, &data, pad, &alphabet); + return_value = binascii_b2a_base32_impl(module, &data, padded, wrapcol, &alphabet); exit: /* Cleanup for data */ @@ -852,7 +1114,7 @@ PyDoc_STRVAR(binascii_b2a_hex__doc__, static PyObject * binascii_b2a_hex_impl(PyObject *module, Py_buffer *data, PyObject *sep, - int bytes_per_sep); + Py_ssize_t bytes_per_sep); static PyObject * binascii_b2a_hex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -889,7 +1151,7 @@ binascii_b2a_hex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; Py_buffer data = {NULL, NULL}; PyObject *sep = NULL; - int bytes_per_sep = 1; + Py_ssize_t bytes_per_sep = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -908,9 +1170,17 @@ binascii_b2a_hex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb goto skip_optional_pos; } } - bytes_per_sep = PyLong_AsInt(args[2]); - if (bytes_per_sep == -1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[2]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + bytes_per_sep = ival; } skip_optional_pos: return_value = binascii_b2a_hex_impl(module, &data, sep, bytes_per_sep); @@ -944,7 +1214,7 @@ PyDoc_STRVAR(binascii_hexlify__doc__, static PyObject * binascii_hexlify_impl(PyObject *module, Py_buffer *data, PyObject *sep, - int bytes_per_sep); + Py_ssize_t bytes_per_sep); static PyObject * binascii_hexlify(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -981,7 +1251,7 @@ binascii_hexlify(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; Py_buffer data = {NULL, NULL}; PyObject *sep = NULL; - int bytes_per_sep = 1; + Py_ssize_t bytes_per_sep = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -1000,9 +1270,17 @@ binascii_hexlify(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb goto skip_optional_pos; } } - bytes_per_sep = PyLong_AsInt(args[2]); - if (bytes_per_sep == -1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[2]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + bytes_per_sep = ival; } skip_optional_pos: return_value = binascii_hexlify_impl(module, &data, sep, bytes_per_sep); @@ -1017,68 +1295,168 @@ binascii_hexlify(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb } PyDoc_STRVAR(binascii_a2b_hex__doc__, -"a2b_hex($module, hexstr, /)\n" +"a2b_hex($module, hexstr, /, *, ignorechars=b\'\')\n" "--\n" "\n" "Binary data of hexadecimal representation.\n" "\n" +" ignorechars\n" +" A byte string containing characters to ignore from the input.\n" +"\n" "hexstr must contain an even number of hex digits (upper or lower case).\n" "This function is also available as \"unhexlify()\"."); #define BINASCII_A2B_HEX_METHODDEF \ - {"a2b_hex", (PyCFunction)binascii_a2b_hex, METH_O, binascii_a2b_hex__doc__}, + {"a2b_hex", _PyCFunction_CAST(binascii_a2b_hex), METH_FASTCALL|METH_KEYWORDS, binascii_a2b_hex__doc__}, static PyObject * -binascii_a2b_hex_impl(PyObject *module, Py_buffer *hexstr); +binascii_a2b_hex_impl(PyObject *module, Py_buffer *hexstr, + Py_buffer *ignorechars); static PyObject * -binascii_a2b_hex(PyObject *module, PyObject *arg) +binascii_a2b_hex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(ignorechars), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "ignorechars", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "a2b_hex", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; Py_buffer hexstr = {NULL, NULL}; + Py_buffer ignorechars = {.buf = "", .obj = NULL, .len = 0}; - if (!ascii_buffer_converter(arg, &hexstr)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { goto exit; } - return_value = binascii_a2b_hex_impl(module, &hexstr); + if (!ascii_buffer_converter(args[0], &hexstr)) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + if (PyObject_GetBuffer(args[1], &ignorechars, PyBUF_SIMPLE) != 0) { + goto exit; + } +skip_optional_kwonly: + return_value = binascii_a2b_hex_impl(module, &hexstr, &ignorechars); exit: /* Cleanup for hexstr */ if (hexstr.obj) PyBuffer_Release(&hexstr); + /* Cleanup for ignorechars */ + if (ignorechars.obj) { + PyBuffer_Release(&ignorechars); + } return return_value; } PyDoc_STRVAR(binascii_unhexlify__doc__, -"unhexlify($module, hexstr, /)\n" +"unhexlify($module, hexstr, /, *, ignorechars=b\'\')\n" "--\n" "\n" "Binary data of hexadecimal representation.\n" "\n" +" ignorechars\n" +" A byte string containing characters to ignore from the input.\n" +"\n" "hexstr must contain an even number of hex digits (upper or lower case)."); #define BINASCII_UNHEXLIFY_METHODDEF \ - {"unhexlify", (PyCFunction)binascii_unhexlify, METH_O, binascii_unhexlify__doc__}, + {"unhexlify", _PyCFunction_CAST(binascii_unhexlify), METH_FASTCALL|METH_KEYWORDS, binascii_unhexlify__doc__}, static PyObject * -binascii_unhexlify_impl(PyObject *module, Py_buffer *hexstr); +binascii_unhexlify_impl(PyObject *module, Py_buffer *hexstr, + Py_buffer *ignorechars); static PyObject * -binascii_unhexlify(PyObject *module, PyObject *arg) +binascii_unhexlify(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(ignorechars), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "ignorechars", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "unhexlify", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; Py_buffer hexstr = {NULL, NULL}; + Py_buffer ignorechars = {.buf = "", .obj = NULL, .len = 0}; - if (!ascii_buffer_converter(arg, &hexstr)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!ascii_buffer_converter(args[0], &hexstr)) { goto exit; } - return_value = binascii_unhexlify_impl(module, &hexstr); + if (!noptargs) { + goto skip_optional_kwonly; + } + if (PyObject_GetBuffer(args[1], &ignorechars, PyBUF_SIMPLE) != 0) { + goto exit; + } +skip_optional_kwonly: + return_value = binascii_unhexlify_impl(module, &hexstr, &ignorechars); exit: /* Cleanup for hexstr */ if (hexstr.obj) PyBuffer_Release(&hexstr); + /* Cleanup for ignorechars */ + if (ignorechars.obj) { + PyBuffer_Release(&ignorechars); + } return return_value; } @@ -1256,4 +1634,4 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return return_value; } -/*[clinic end generated code: output=84c97096b0fb3819 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2acab1ceb0058b1a input=a9049054013a1b77]*/ diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 65fbcf5cdaa73f..1e9f9ae051a0b1 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -1293,6 +1293,7 @@ cmath_exec(PyObject *mod) } static PyModuleDef_Slot cmath_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, cmath_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 6c7c4186927725..18dbaebde293bc 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -1082,7 +1082,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); */ # define XML_MAJOR_VERSION 2 # define XML_MINOR_VERSION 7 -# define XML_MICRO_VERSION 4 +# define XML_MICRO_VERSION 5 # ifdef __cplusplus } diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index 6f3f3c48ce9cff..cf4d445e68b00c 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -12,7 +12,7 @@ Copyright (c) 2001-2002 Greg Stein Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2016 Cristian Rodríguez - Copyright (c) 2016-2025 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2017 Rhodri James Copyright (c) 2018 Yury Gribov Licensed under the MIT license: diff --git a/Modules/expat/refresh.sh b/Modules/expat/refresh.sh index 0e0bc0652c5539..779929fc6ed33c 100755 --- a/Modules/expat/refresh.sh +++ b/Modules/expat/refresh.sh @@ -12,9 +12,9 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. These values are used for verifying the SBOM, too. -expected_libexpat_tag="R_2_7_4" -expected_libexpat_version="2.7.4" -expected_libexpat_sha256="461ecc8aa98ab1a68c2db788175665d1a4db640dc05bf0e289b6ea17122144ec" +expected_libexpat_tag="R_2_7_5" +expected_libexpat_version="2.7.5" +expected_libexpat_sha256="9931f9860d18e6cf72d183eb8f309bfb96196c00e1d40caa978e95bc9aa978b6" expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")" cd ${expat_dir} diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 086fca59112ee1..0248b6651ffbff 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* fab937ab8b186d7d296013669c332e6dfce2f99567882cff1f8eb24223c524a7 (2.7.4+) +/* 93c1caa66e2b0310459482516af05505b57c5cb7b96df777105308fc585c85d1 (2.7.5+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -590,6 +590,8 @@ static XML_Char *poolStoreString(STRING_POOL *pool, const ENCODING *enc, static XML_Bool FASTCALL poolGrow(STRING_POOL *pool); static const XML_Char *FASTCALL poolCopyString(STRING_POOL *pool, const XML_Char *s); +static const XML_Char *FASTCALL poolCopyStringNoFinish(STRING_POOL *pool, + const XML_Char *s); static const XML_Char *poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n); static const XML_Char *FASTCALL poolAppendString(STRING_POOL *pool, @@ -5086,7 +5088,7 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, } /* If we get this token, we have the start of what might be a normal tag, but not a declaration (i.e. it doesn't begin with - "= entityTextEnd) { + result = XML_ERROR_NONE; + goto endEntityValue; + } + for (;;) { next = entityTextPtr; /* XmlEntityValueTok doesn't always set the last arg */ @@ -7439,16 +7457,24 @@ setContext(XML_Parser parser, const XML_Char *context) { else { if (! poolAppendChar(&parser->m_tempPool, XML_T('\0'))) return XML_FALSE; - prefix - = (PREFIX *)lookup(parser, &dtd->prefixes, - poolStart(&parser->m_tempPool), sizeof(PREFIX)); - if (! prefix) + const XML_Char *const prefixName = poolCopyStringNoFinish( + &dtd->pool, poolStart(&parser->m_tempPool)); + if (! prefixName) { return XML_FALSE; - if (prefix->name == poolStart(&parser->m_tempPool)) { - prefix->name = poolCopyString(&dtd->pool, prefix->name); - if (! prefix->name) - return XML_FALSE; } + + prefix = (PREFIX *)lookup(parser, &dtd->prefixes, prefixName, + sizeof(PREFIX)); + + const bool prefixNameUsed = prefix && prefix->name == prefixName; + if (prefixNameUsed) + poolFinish(&dtd->pool); + else + poolDiscard(&dtd->pool); + + if (! prefix) + return XML_FALSE; + poolDiscard(&parser->m_tempPool); } for (context = s + 1; *context != CONTEXT_SEP && *context != XML_T('\0'); @@ -8036,6 +8062,23 @@ poolCopyString(STRING_POOL *pool, const XML_Char *s) { return s; } +// A version of `poolCopyString` that does not call `poolFinish` +// and reverts any partial advancement upon failure. +static const XML_Char *FASTCALL +poolCopyStringNoFinish(STRING_POOL *pool, const XML_Char *s) { + const XML_Char *const original = s; + do { + if (! poolAppendChar(pool, *s)) { + // Revert any previously successful advancement + const ptrdiff_t advancedBy = s - original; + if (advancedBy > 0) + pool->ptr -= advancedBy; + return NULL; + } + } while (*s++); + return pool->start; +} + static const XML_Char * poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { if (! pool->ptr && ! poolGrow(pool)) { diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index d56bee82dd2d13..b1dfb456e5df87 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -12,7 +12,7 @@ Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2002-2003 Fred L. Drake, Jr. Copyright (c) 2005-2009 Steven Solie - Copyright (c) 2016-2023 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2017 Rhodri James Copyright (c) 2019 David Loffredo Copyright (c) 2021 Donghee Na diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index 32cd5f147e9322..f6e5f742c928c8 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -12,7 +12,7 @@ Copyright (c) 2002 Greg Stein Copyright (c) 2002-2016 Karl Waclawek Copyright (c) 2005-2009 Steven Solie - Copyright (c) 2016-2024 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2016 Pascal Cuoq Copyright (c) 2016 Don Lewis Copyright (c) 2017 Rhodri James diff --git a/Modules/expat/xmltok_ns.c b/Modules/expat/xmltok_ns.c index 810ca2c6d0485e..1cd60de1e4fe51 100644 --- a/Modules/expat/xmltok_ns.c +++ b/Modules/expat/xmltok_ns.c @@ -11,7 +11,7 @@ Copyright (c) 2002 Greg Stein Copyright (c) 2002 Fred L. Drake, Jr. Copyright (c) 2002-2006 Karl Waclawek - Copyright (c) 2017-2021 Sebastian Pipping + Copyright (c) 2017-2026 Sebastian Pipping Copyright (c) 2025 Alfonso Gregory Licensed under the MIT license: diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 9b8c77e2b0401f..437575ea6b793f 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -7,7 +7,7 @@ #include "pycore_runtime.h" // _Py_ID() #include "pycore_signal.h" // Py_NSIG #include "pycore_time.h" // _PyTime_FromSecondsObject() -#include "pycore_traceback.h" // _Py_DumpTracebackThreads +#include "pycore_traceback.h" // _Py_DumpStack() #ifdef HAVE_UNISTD_H # include // _exit() #endif @@ -205,14 +205,15 @@ faulthandler_dump_traceback(int fd, int all_threads, PyThreadState *tstate = PyGILState_GetThisThreadState(); if (all_threads == 1) { - (void)_Py_DumpTracebackThreads(fd, NULL, tstate); + (void)PyUnstable_DumpTracebackThreads(fd, NULL, tstate); } else { if (all_threads == FT_IGNORE_ALL_THREADS) { PUTS(fd, "\n"); } - if (tstate != NULL) - _Py_DumpTraceback(fd, tstate); + if (tstate != NULL) { + PyUnstable_DumpTraceback(fd, tstate); + } } reentrant = 0; @@ -273,17 +274,18 @@ faulthandler_dump_traceback_py_impl(PyObject *module, PyObject *file, /* gh-128400: Accessing other thread states while they're running * isn't safe if those threads are running. */ _PyEval_StopTheWorld(interp); - errmsg = _Py_DumpTracebackThreads(fd, NULL, tstate); + errmsg = PyUnstable_DumpTracebackThreads(fd, NULL, tstate); _PyEval_StartTheWorld(interp); - if (errmsg != NULL) { - PyErr_SetString(PyExc_RuntimeError, errmsg); - Py_XDECREF(file); - return NULL; - } } else { - _Py_DumpTraceback(fd, tstate); + errmsg = PyUnstable_DumpTraceback(fd, tstate); + } + if (errmsg != NULL) { + PyErr_SetString(PyExc_RuntimeError, errmsg); + Py_XDECREF(file); + return NULL; } + Py_XDECREF(file); if (PyErr_CheckSignals()) @@ -703,7 +705,7 @@ faulthandler_thread(void *unused) (void)_Py_write_noraise(thread.fd, thread.header, (int)thread.header_len); - errmsg = _Py_DumpTracebackThreads(thread.fd, thread.interp, NULL); + errmsg = PyUnstable_DumpTracebackThreads(thread.fd, thread.interp, NULL); ok = (errmsg == NULL); if (thread.exit) @@ -1422,6 +1424,7 @@ PyExec_faulthandler(PyObject *module) { } static PyModuleDef_Slot faulthandler_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, PyExec_faulthandler}, // XXX gh-103092: fix isolation. //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index ce636c574ed5ff..e6a40ffc5a2614 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -835,6 +835,7 @@ fcntl_exec(PyObject *module) } static PyModuleDef_Slot fcntl_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, fcntl_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 4c286f5c12cc7d..8da28130e9da9a 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -347,9 +347,9 @@ gc_get_stats_impl(PyObject *module) /* To get consistent values despite allocations while constructing the result list, we use a snapshot of the running stats. */ GCState *gcstate = get_gc_state(); - for (i = 0; i < NUM_GENERATIONS; i++) { - stats[i] = gcstate->generation_stats[i]; - } + stats[0] = gcstate->generation_stats->young.items[gcstate->generation_stats->young.index]; + stats[1] = gcstate->generation_stats->old[0].items[gcstate->generation_stats->old[0].index]; + stats[2] = gcstate->generation_stats->old[1].items[gcstate->generation_stats->old[1].index]; PyObject *result = PyList_New(0); if (result == NULL) @@ -538,6 +538,7 @@ gcmodule_exec(PyObject *module) } static PyModuleDef_Slot gcmodule_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, gcmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index 652958618a2c4c..32ead259803614 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -370,6 +370,7 @@ grpmodule_exec(PyObject *module) } static PyModuleDef_Slot grpmodule_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, grpmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/hmacmodule.c b/Modules/hmacmodule.c index 1a212fa3d37e18..b39a8f99ed91e8 100644 --- a/Modules/hmacmodule.c +++ b/Modules/hmacmodule.c @@ -1690,6 +1690,7 @@ hmacmodule_free(void *mod) } static struct PyModuleDef_Slot hmacmodule_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, hmacmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index b37256c7928bad..a6bfa78a461bb0 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -310,7 +310,7 @@ pairwise_new_impl(PyTypeObject *type, PyObject *iterable) } po->it = it; po->old = NULL; - po->result = PyTuple_Pack(2, Py_None, Py_None); + po->result = _PyTuple_FromPairSteal(Py_None, Py_None); if (po->result == NULL) { Py_DECREF(po); return NULL; @@ -389,11 +389,7 @@ pairwise_next(PyObject *op) _PyTuple_Recycle(result); } else { - result = PyTuple_New(2); - if (result != NULL) { - PyTuple_SET_ITEM(result, 0, Py_NewRef(old)); - PyTuple_SET_ITEM(result, 1, Py_NewRef(new)); - } + result = _PyTuple_FromPair(old, new); } Py_XSETREF(po->old, new); @@ -533,7 +529,7 @@ groupby_step(groupbyobject *gbo) static PyObject * groupby_next(PyObject *op) { - PyObject *r, *grouper; + PyObject *grouper; groupbyobject *gbo = groupbyobject_CAST(op); gbo->currgrouper = NULL; @@ -573,9 +569,7 @@ groupby_next(PyObject *op) if (grouper == NULL) return NULL; - r = PyTuple_Pack(2, gbo->currkey, grouper); - Py_DECREF(grouper); - return r; + return _PyTuple_FromPairSteal(Py_NewRef(gbo->currkey), grouper); } static PyType_Slot groupby_slots[] = { @@ -678,7 +672,16 @@ _grouper_next(PyObject *op) } assert(gbo->currkey != NULL); - rcmp = PyObject_RichCompareBool(igo->tgtkey, gbo->currkey, Py_EQ); + /* A user-defined __eq__ can re-enter the grouper and advance the iterator, + mutating gbo->currkey while we are comparing them. + Take local snapshots and hold strong references so INCREF/DECREF + apply to the same objects even under re-entrancy. */ + PyObject *tgtkey = Py_NewRef(igo->tgtkey); + PyObject *currkey = Py_NewRef(gbo->currkey); + rcmp = PyObject_RichCompareBool(tgtkey, currkey, Py_EQ); + Py_DECREF(tgtkey); + Py_DECREF(currkey); + if (rcmp <= 0) /* got any error or current group is end */ return NULL; @@ -3876,7 +3879,7 @@ zip_longest_traverse(PyObject *op, visitproc visit, void *arg) } static PyObject * -zip_longest_next(PyObject *op) +zip_longest_next_lock_held(PyObject *op) { ziplongestobject *lz = ziplongestobject_CAST(op); Py_ssize_t i; @@ -3947,6 +3950,16 @@ zip_longest_next(PyObject *op) return result; } +static PyObject * +zip_longest_next(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = zip_longest_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; +} + PyDoc_STRVAR(zip_longest_doc, "zip_longest(*iterables, fillvalue=None)\n\ --\n\ @@ -4121,6 +4134,7 @@ itertoolsmodule_exec(PyObject *mod) } static struct PyModuleDef_Slot itertoolsmodule_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, itertoolsmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/main.c b/Modules/main.c index 7731fa0c7c6717..a4dfddd98e257e 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -11,6 +11,7 @@ #include "pycore_pylifecycle.h" // _Py_PreInitializeFromPyArgv() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_pythonrun.h" // _PyRun_AnyFileObject() +#include "pycore_tuple.h" // _PyTuple_FromPair #include "pycore_unicodeobject.h" // _PyUnicode_Dedent() /* Includes for exit_sigint() */ @@ -342,7 +343,7 @@ pymain_run_module(const wchar_t *modname, int set_argv0) Py_DECREF(runmodule); return pymain_exit_err_print(); } - runargs = PyTuple_Pack(2, module, set_argv0 ? Py_True : Py_False); + runargs = _PyTuple_FromPair(module, set_argv0 ? Py_True : Py_False); if (runargs == NULL) { fprintf(stderr, "Could not create arguments for runpy._run_module_as_main\n"); diff --git a/Modules/mathintegermodule.c b/Modules/mathintegermodule.c index de5f619c9d065c..cfad4154b2d361 100644 --- a/Modules/mathintegermodule.c +++ b/Modules/mathintegermodule.c @@ -1268,6 +1268,7 @@ math_integer_exec(PyObject *module) } static PyModuleDef_Slot math_integer_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, math_integer_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 11c46c987e146a..6b7fc004d0d858 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -3103,6 +3103,7 @@ static PyMethodDef math_methods[] = { }; static PyModuleDef_Slot math_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, math_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/md5module.c b/Modules/md5module.c index e598b1fe67240d..063be1405dd51f 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -365,6 +365,7 @@ md5_exec(PyObject *m) } static PyModuleDef_Slot _md5_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, md5_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 61d8a043a04ce2..a30afe91f8fa17 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -2432,6 +2432,7 @@ mmap_exec(PyObject *module) } static PyModuleDef_Slot mmap_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, mmap_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/overlapped.c b/Modules/overlapped.c index 8c3575ff5678eb..822e1ce4bdc28d 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -12,6 +12,7 @@ #endif #include "Python.h" +#include "pycore_tuple.h" // _PyTuple_FromPairSteal #define WINDOWS_LEAN_AND_MEAN #include @@ -896,6 +897,7 @@ _overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait) BOOL ret; DWORD err; PyObject *addr; + PyObject *transferred_obj; if (self->type == TYPE_NONE) { PyErr_SetString(PyExc_ValueError, "operation not yet attempted"); @@ -964,18 +966,12 @@ _overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait) } // The result is a two item tuple: (message, address) - self->read_from.result = PyTuple_New(2); + self->read_from.result = _PyTuple_FromPairSteal( + Py_NewRef(self->read_from.allocated_buffer), addr); if (self->read_from.result == NULL) { - Py_CLEAR(addr); return NULL; } - // first item: message - PyTuple_SET_ITEM(self->read_from.result, 0, - Py_NewRef(self->read_from.allocated_buffer)); - // second item: address - PyTuple_SET_ITEM(self->read_from.result, 1, addr); - return Py_NewRef(self->read_from.result); case TYPE_READ_FROM_INTO: // unparse the address @@ -986,19 +982,19 @@ _overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait) return NULL; } + transferred_obj = PyLong_FromUnsignedLong((unsigned long)transferred); + if (transferred_obj == NULL) { + Py_DECREF(addr); + return NULL; + } + // The result is a two item tuple: (number of bytes read, address) - self->read_from_into.result = PyTuple_New(2); + self->read_from_into.result = _PyTuple_FromPairSteal( + transferred_obj, addr); if (self->read_from_into.result == NULL) { - Py_CLEAR(addr); return NULL; } - // first item: number of bytes read - PyTuple_SET_ITEM(self->read_from_into.result, 0, - PyLong_FromUnsignedLong((unsigned long)transferred)); - // second item: address - PyTuple_SET_ITEM(self->read_from_into.result, 1, addr); - return Py_NewRef(self->read_from_into.result); default: return PyLong_FromUnsignedLong((unsigned long) transferred); @@ -2073,6 +2069,7 @@ overlapped_exec(PyObject *module) } static PyModuleDef_Slot overlapped_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, overlapped_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 157965195e1fa0..e5ce487723b25b 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -27,6 +27,7 @@ #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_signal.h" // Py_NSIG #include "pycore_time.h" // _PyLong_FromTime_t() +#include "pycore_tuple.h" // _PyTuple_FromPairSteal #include "pycore_typeobject.h" // _PyType_AddMethod() #ifndef MS_WINDOWS @@ -11288,10 +11289,7 @@ build_itimerspec(const struct itimerspec* curr_value) Py_DECREF(value); return NULL; } - PyObject *tuple = PyTuple_Pack(2, value, interval); - Py_DECREF(interval); - Py_DECREF(value); - return tuple; + return _PyTuple_FromPairSteal(value, interval); } static PyObject * @@ -18815,6 +18813,7 @@ posixmodule_exec(PyObject *m) static PyModuleDef_Slot posixmodile_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, posixmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c index a18737b24c29e9..4a2b33f8700d10 100644 --- a/Modules/pwdmodule.c +++ b/Modules/pwdmodule.c @@ -372,6 +372,7 @@ pwdmodule_exec(PyObject *module) } static PyModuleDef_Slot pwdmodule_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, pwdmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 782e552f342b17..31b883fe8bd548 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -2464,6 +2464,7 @@ pyexpat_free(void *module) } static PyModuleDef_Slot pyexpat_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, pyexpat_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/readline.c b/Modules/readline.c index 579a34b02ceb67..488332f548e5fe 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -1626,6 +1626,11 @@ static struct PyModuleDef readlinemodule = { PyMODINIT_FUNC PyInit_readline(void) { + PyABIInfo_VAR(abi_info); + if (PyABIInfo_Check(&abi_info, "readline") < 0) { + return NULL; + } + const char *backend = "readline"; PyObject *m; readlinestate *mod_state; diff --git a/Modules/resource.c b/Modules/resource.c index a463355f424d48..9bf8d2782766cc 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -560,6 +560,7 @@ resource_exec(PyObject *module) } static struct PyModuleDef_Slot resource_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, resource_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 4dd544c6ee8d34..eb3148ef24631b 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -15,6 +15,7 @@ #include "Python.h" #include "pycore_fileutils.h" // _Py_set_inheritable() #include "pycore_time.h" // _PyTime_FromSecondsObject() +#include "pycore_tuple.h" // _PyTuple_FromPairSteal #include #include // offsetof() @@ -1075,9 +1076,7 @@ select_devpoll_poll_impl(devpollObject *self, PyObject *timeout_obj) Py_XDECREF(num2); goto error; } - value = PyTuple_Pack(2, num1, num2); - Py_DECREF(num1); - Py_DECREF(num2); + value = _PyTuple_FromPairSteal(num1, num2); if (value == NULL) goto error; PyList_SET_ITEM(result_list, i, value); @@ -2912,6 +2911,7 @@ _select_exec(PyObject *m) } static PyModuleDef_Slot _select_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _select_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/sha1module.c b/Modules/sha1module.c index 89e66240d1d11f..5681780b569b6c 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -369,6 +369,7 @@ _sha1_exec(PyObject *module) /* Initialize this module. */ static PyModuleDef_Slot _sha1_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _sha1_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/sha2module.c b/Modules/sha2module.c index 9453b0be512555..7613ee54954dd6 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -883,6 +883,7 @@ static int sha2_exec(PyObject *module) } static PyModuleDef_Slot _sha2_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, sha2_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/sha3module.c b/Modules/sha3module.c index 38c9bc0405be60..3ddd0323575b70 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -680,6 +680,7 @@ _sha3_exec(PyObject *m) } static PyModuleDef_Slot _sha3_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _sha3_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 5060e4097d33c9..fb548b8ca00f24 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -14,6 +14,7 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_signal.h" // _Py_RestoreSignals() #include "pycore_time.h" // _PyTime_FromSecondsObject() +#include "pycore_tuple.h" // _PyTuple_FromPairSteal #ifndef MS_WINDOWS # include "posixmodule.h" // _PyLong_FromUid() @@ -193,27 +194,16 @@ double_from_timeval(struct timeval *tv) static PyObject * itimer_retval(struct itimerval *iv) { - PyObject *r, *v; - - r = PyTuple_New(2); - if (r == NULL) - return NULL; - - if(!(v = PyFloat_FromDouble(double_from_timeval(&iv->it_value)))) { - Py_DECREF(r); + PyObject *value = PyFloat_FromDouble(double_from_timeval(&iv->it_value)); + if (value == NULL) { return NULL; } - - PyTuple_SET_ITEM(r, 0, v); - - if(!(v = PyFloat_FromDouble(double_from_timeval(&iv->it_interval)))) { - Py_DECREF(r); + PyObject *interval = PyFloat_FromDouble(double_from_timeval(&iv->it_interval)); + if (interval == NULL) { + Py_DECREF(value); return NULL; } - - PyTuple_SET_ITEM(r, 1, v); - - return r; + return _PyTuple_FromPairSteal(value, interval); } #endif @@ -1709,6 +1699,7 @@ _signal_module_free(void *module) static PyModuleDef_Slot signal_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, signal_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index d4df40c78e8a4f..f1a55db229e115 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -111,6 +111,7 @@ Local naming conventions: #include "pycore_moduleobject.h" // _PyModule_GetState #include "pycore_object.h" // _PyObject_VisitType() #include "pycore_time.h" // _PyTime_AsMilliseconds() +#include "pycore_tuple.h" // _PyTuple_FromPairSteal #include "pycore_pystate.h" // _Py_AssertHoldsTstate() #ifdef _Py_MEMORY_SANITIZER @@ -151,7 +152,7 @@ listen([n]) -- start listening for incoming connections\n\ recv(buflen[, flags]) -- receive data\n\ recv_into(buffer[, nbytes[, flags]]) -- receive data (into a buffer)\n\ recvfrom(buflen[, flags]) -- receive data and sender\'s address\n\ -recvfrom_into(buffer[, nbytes, [, flags])\n\ +recvfrom_into(buffer[, nbytes, [, flags]])\n\ -- receive data and sender\'s address (into a buffer)\n\ sendall(data[, flags]) -- send all data\n\ send(data[, flags]) -- send data, may not send all of it\n\ @@ -3076,7 +3077,6 @@ sock_accept(PyObject *self, PyObject *Py_UNUSED(ignored)) socklen_t addrlen; PyObject *sock = NULL; PyObject *addr = NULL; - PyObject *res = NULL; struct sock_accept ctx; if (!getsockaddrlen(s, &addrlen)) @@ -3102,7 +3102,7 @@ sock_accept(PyObject *self, PyObject *Py_UNUSED(ignored)) if (!SetHandleInformation((HANDLE)newfd, HANDLE_FLAG_INHERIT, 0)) { PyErr_SetFromWindowsErr(0); SOCKETCLOSE(newfd); - goto finally; + goto error; } #endif #else @@ -3113,7 +3113,7 @@ sock_accept(PyObject *self, PyObject *Py_UNUSED(ignored)) { if (_Py_set_inheritable(newfd, 0, NULL) < 0) { SOCKETCLOSE(newfd); - goto finally; + goto error; } } #endif @@ -3121,20 +3121,20 @@ sock_accept(PyObject *self, PyObject *Py_UNUSED(ignored)) sock = PyLong_FromSocket_t(newfd); if (sock == NULL) { SOCKETCLOSE(newfd); - goto finally; + goto error; } addr = makesockaddr(get_sock_fd(s), SAS2SA(&addrbuf), addrlen, s->sock_proto); if (addr == NULL) - goto finally; + goto error; - res = PyTuple_Pack(2, sock, addr); + return _PyTuple_FromPairSteal(sock, addr); -finally: +error: Py_XDECREF(sock); Py_XDECREF(addr); - return res; + return NULL; } PyDoc_STRVAR(accept_doc, @@ -3357,8 +3357,7 @@ sock_setsockopt(PyObject *self, PyObject *args) arglen = PyTuple_Size(args); if (arglen == 3 && optval == Py_None) { PyErr_Format(PyExc_TypeError, - "setsockopt() requires 4 arguments when the third argument is None", - arglen); + "setsockopt() requires 4 arguments when the third argument is None"); return NULL; } if (arglen == 4 && optval != Py_None) { @@ -4167,7 +4166,6 @@ sock_recvfrom(PyObject *self, PyObject *args) PySocketSockObject *s = _PySocketSockObject_CAST(self); PyObject *addr = NULL; - PyObject *ret = NULL; int flags = 0; Py_ssize_t recvlen, outlen; @@ -4189,20 +4187,19 @@ sock_recvfrom(PyObject *self, PyObject *args) recvlen, flags, &addr); if (outlen < 0) { PyBytesWriter_Discard(writer); - goto finally; + goto error; } PyObject *buf = PyBytesWriter_FinishWithSize(writer, outlen); if (buf == NULL) { - goto finally; + goto error; } - ret = PyTuple_Pack(2, buf, addr); - Py_DECREF(buf); + return _PyTuple_FromPairSteal(buf, addr); -finally: +error: Py_XDECREF(addr); - return ret; + return NULL; } PyDoc_STRVAR(recvfrom_doc, @@ -4808,6 +4805,7 @@ sock_sendto(PyObject *self, PyObject *args) } if (PySys_Audit("socket.sendto", "OO", s, addro) < 0) { + PyBuffer_Release(&pbuf); return NULL; } @@ -6261,7 +6259,7 @@ socket_gethostbyaddr(PyObject *self, PyObject *args) gethostbyaddr_r is 8-byte aligned, which at least llvm-gcc does not ensure. The attribute below instructs the compiler to maintain this alignment. */ - char buf[16384] Py_ALIGNED(8); + _Py_ALIGNED_DEF(8, char) buf[16384]; int buf_len = (sizeof buf) - 1; int errnop; #endif @@ -6543,7 +6541,6 @@ socket_socketpair(PyObject *self, PyObject *args) PySocketSockObject *s0 = NULL, *s1 = NULL; SOCKET_T sv[2]; int family, type = SOCK_STREAM, proto = 0; - PyObject *res = NULL; socket_state *state = get_module_state(self); #ifdef SOCK_CLOEXEC int *atomic_flag_works = &sock_cloexec_works; @@ -6588,28 +6585,26 @@ socket_socketpair(PyObject *self, PyObject *args) return set_error(); if (_Py_set_inheritable(sv[0], 0, atomic_flag_works) < 0) - goto finally; + goto error; if (_Py_set_inheritable(sv[1], 0, atomic_flag_works) < 0) - goto finally; + goto error; s0 = new_sockobject(state, sv[0], family, type, proto); if (s0 == NULL) - goto finally; + goto error; s1 = new_sockobject(state, sv[1], family, type, proto); if (s1 == NULL) - goto finally; - res = PyTuple_Pack(2, s0, s1); + goto error; + return _PyTuple_FromPairSteal((PyObject *)s0, (PyObject *)s1); -finally: - if (res == NULL) { - if (s0 == NULL) - SOCKETCLOSE(sv[0]); - if (s1 == NULL) - SOCKETCLOSE(sv[1]); - } +error: + if (s0 == NULL) + SOCKETCLOSE(sv[0]); + if (s1 == NULL) + SOCKETCLOSE(sv[1]); Py_XDECREF(s0); Py_XDECREF(s1); - return res; + return NULL; } PyDoc_STRVAR(socketpair_doc, @@ -6982,7 +6977,7 @@ socket_getaddrinfo(PyObject *self, PyObject *args, PyObject* kwargs) if (PySys_Audit("socket.getaddrinfo", "OOiii", hobj, pobj, family, socktype, protocol) < 0) { - return NULL; + goto err; } memset(&hints, 0, sizeof(hints)); @@ -9305,6 +9300,7 @@ socket_exec(PyObject *m) } static struct PyModuleDef_Slot socket_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, socket_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index a24927a9db64db..e9e1c4811b8303 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -122,6 +122,7 @@ symtable_init_constants(PyObject *m) } static PyModuleDef_Slot symtable_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, symtable_init_constants}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index 5d7fd20c4e0999..2d13f9eda758dd 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -451,6 +451,7 @@ syslog_exec(PyObject *module) } static PyModuleDef_Slot syslog_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, syslog_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/timemodule.c b/Modules/timemodule.c index a3260e0f15ab99..25e744d7da25c7 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -2185,6 +2185,7 @@ time_module_free(void *module) static struct PyModuleDef_Slot time_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, time_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index 2c67c23d98ed81..55b33a76e7af8a 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -2303,6 +2303,7 @@ unicodedata_exec(PyObject *module) } static PyModuleDef_Slot unicodedata_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, unicodedata_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/xxmodule.c b/Modules/xxmodule.c index e8749331c6a11f..aeab78fd77d83b 100644 --- a/Modules/xxmodule.c +++ b/Modules/xxmodule.c @@ -386,6 +386,7 @@ xx_exec(PyObject *m) } static struct PyModuleDef_Slot xx_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, xx_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/xxsubtype.c b/Modules/xxsubtype.c index a8a1417f40efef..7a31ba00b981eb 100644 --- a/Modules/xxsubtype.c +++ b/Modules/xxsubtype.c @@ -301,7 +301,10 @@ xxsubtype_exec(PyObject* m) return 0; } +PyABIInfo_VAR(abi_info); + static struct PyModuleDef_Slot xxsubtype_slots[] = { + {Py_mod_abi, &abi_info}, {Py_mod_exec, xxsubtype_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 7a8ed979bbe65d..f67434ecdc908c 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -2272,6 +2272,7 @@ zlib_exec(PyObject *mod) } static PyModuleDef_Slot zlib_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, zlib_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index e2fea94e099626..c583193b5a252c 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -1752,27 +1752,26 @@ bytearray_maketrans_impl(Py_buffer *frm, Py_buffer *to) /*[clinic input] -@permit_long_docstring_body @critical_section bytearray.replace old: Py_buffer new: Py_buffer + / count: Py_ssize_t = -1 Maximum number of occurrences to replace. -1 (the default value) means replace all occurrences. - / Return a copy with all occurrences of substring old replaced by new. -If the optional argument count is given, only the first count occurrences are -replaced. +If count is given, only the first count occurrences are replaced. +If count is not specified or -1, then all occurrences are replaced. [clinic start generated code]*/ static PyObject * bytearray_replace_impl(PyByteArrayObject *self, Py_buffer *old, Py_buffer *new, Py_ssize_t count) -/*[clinic end generated code: output=d39884c4dc59412a input=66afec32f4e095e0]*/ +/*[clinic end generated code: output=d39884c4dc59412a input=e2591806f954aec3]*/ { return stringlib_replace((PyObject *)self, (const char *)old->buf, old->len, @@ -2641,7 +2640,7 @@ bytearray.hex sep: object = NULL An optional single character or byte to separate hex bytes. - bytes_per_sep: int = 1 + bytes_per_sep: Py_ssize_t = 1 How many bytes between separators. Positive values count from the right, negative values count from the left. @@ -2660,8 +2659,9 @@ Create a string of hexadecimal numbers from a bytearray object. [clinic start generated code]*/ static PyObject * -bytearray_hex_impl(PyByteArrayObject *self, PyObject *sep, int bytes_per_sep) -/*[clinic end generated code: output=29c4e5ef72c565a0 input=7784107de7048873]*/ +bytearray_hex_impl(PyByteArrayObject *self, PyObject *sep, + Py_ssize_t bytes_per_sep) +/*[clinic end generated code: output=c9563921aff1262b input=d2b23ef057cfcad5]*/ { char* argbuf = PyByteArray_AS_STRING(self); Py_ssize_t arglen = PyByteArray_GET_SIZE(self); diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 00c1c63b8e01c6..902144e8ec9f83 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -2403,26 +2403,25 @@ bytes_maketrans_impl(Py_buffer *frm, Py_buffer *to) /*[clinic input] -@permit_long_docstring_body bytes.replace old: Py_buffer new: Py_buffer + / count: Py_ssize_t = -1 Maximum number of occurrences to replace. -1 (the default value) means replace all occurrences. - / Return a copy with all occurrences of substring old replaced by new. -If the optional argument count is given, only the first count occurrences are -replaced. +If count is given, only the first count occurrences are replaced. +If count is not specified or -1, then all occurrences are replaced. [clinic start generated code]*/ static PyObject * bytes_replace_impl(PyBytesObject *self, Py_buffer *old, Py_buffer *new, Py_ssize_t count) -/*[clinic end generated code: output=994fa588b6b9c104 input=8b99a9ab32bc06a2]*/ +/*[clinic end generated code: output=994fa588b6b9c104 input=cdf3cf8639297745]*/ { return stringlib_replace((PyObject *)self, (const char *)old->buf, old->len, @@ -2744,7 +2743,7 @@ bytes.hex sep: object = NULL An optional single character or byte to separate hex bytes. - bytes_per_sep: int = 1 + bytes_per_sep: Py_ssize_t = 1 How many bytes between separators. Positive values count from the right, negative values count from the left. @@ -2763,8 +2762,8 @@ Create a string of hexadecimal numbers from a bytes object. [clinic start generated code]*/ static PyObject * -bytes_hex_impl(PyBytesObject *self, PyObject *sep, int bytes_per_sep) -/*[clinic end generated code: output=1f134da504064139 input=1a21282b1f1ae595]*/ +bytes_hex_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t bytes_per_sep) +/*[clinic end generated code: output=588821f02cb9d8f5 input=bd8eceb755d8230f]*/ { const char *argbuf = PyBytes_AS_STRING(self); Py_ssize_t arglen = PyBytes_GET_SIZE(self); diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index cf60d0ceadc7d1..64603adcc1124b 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -793,7 +793,7 @@ bytearray_maketrans(PyObject *null, PyObject *const *args, Py_ssize_t nargs) } PyDoc_STRVAR(bytearray_replace__doc__, -"replace($self, old, new, count=-1, /)\n" +"replace($self, old, new, /, count=-1)\n" "--\n" "\n" "Return a copy with all occurrences of substring old replaced by new.\n" @@ -802,25 +802,56 @@ PyDoc_STRVAR(bytearray_replace__doc__, " Maximum number of occurrences to replace.\n" " -1 (the default value) means replace all occurrences.\n" "\n" -"If the optional argument count is given, only the first count occurrences are\n" -"replaced."); +"If count is given, only the first count occurrences are replaced.\n" +"If count is not specified or -1, then all occurrences are replaced."); #define BYTEARRAY_REPLACE_METHODDEF \ - {"replace", _PyCFunction_CAST(bytearray_replace), METH_FASTCALL, bytearray_replace__doc__}, + {"replace", _PyCFunction_CAST(bytearray_replace), METH_FASTCALL|METH_KEYWORDS, bytearray_replace__doc__}, static PyObject * bytearray_replace_impl(PyByteArrayObject *self, Py_buffer *old, Py_buffer *new, Py_ssize_t count); static PyObject * -bytearray_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +bytearray_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(count), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "", "count", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "replace", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; Py_buffer old = {NULL, NULL}; Py_buffer new = {NULL, NULL}; Py_ssize_t count = -1; - if (!_PyArg_CheckPositional("replace", nargs, 2, 3)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { goto exit; } if (PyObject_GetBuffer(args[0], &old, PyBUF_SIMPLE) != 0) { @@ -829,8 +860,8 @@ bytearray_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[1], &new, PyBUF_SIMPLE) != 0) { goto exit; } - if (nargs < 3) { - goto skip_optional; + if (!noptargs) { + goto skip_optional_pos; } { Py_ssize_t ival = -1; @@ -844,7 +875,7 @@ bytearray_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs) } count = ival; } -skip_optional: +skip_optional_pos: Py_BEGIN_CRITICAL_SECTION(self); return_value = bytearray_replace_impl((PyByteArrayObject *)self, &old, &new, count); Py_END_CRITICAL_SECTION(); @@ -1692,7 +1723,8 @@ PyDoc_STRVAR(bytearray_hex__doc__, {"hex", _PyCFunction_CAST(bytearray_hex), METH_FASTCALL|METH_KEYWORDS, bytearray_hex__doc__}, static PyObject * -bytearray_hex_impl(PyByteArrayObject *self, PyObject *sep, int bytes_per_sep); +bytearray_hex_impl(PyByteArrayObject *self, PyObject *sep, + Py_ssize_t bytes_per_sep); static PyObject * bytearray_hex(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -1728,7 +1760,7 @@ bytearray_hex(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *sep = NULL; - int bytes_per_sep = 1; + Py_ssize_t bytes_per_sep = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -1744,9 +1776,17 @@ bytearray_hex(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_pos; } } - bytes_per_sep = PyLong_AsInt(args[1]); - if (bytes_per_sep == -1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + bytes_per_sep = ival; } skip_optional_pos: Py_BEGIN_CRITICAL_SECTION(self); @@ -1835,4 +1875,4 @@ bytearray_sizeof(PyObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl((PyByteArrayObject *)self); } -/*[clinic end generated code: output=2d76ef023928424f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2cacb323147202b9 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/bytesobject.c.h b/Objects/clinic/bytesobject.c.h index 00cf13d422d900..4ff696be91b12d 100644 --- a/Objects/clinic/bytesobject.c.h +++ b/Objects/clinic/bytesobject.c.h @@ -789,7 +789,7 @@ bytes_maketrans(PyObject *null, PyObject *const *args, Py_ssize_t nargs) } PyDoc_STRVAR(bytes_replace__doc__, -"replace($self, old, new, count=-1, /)\n" +"replace($self, old, new, /, count=-1)\n" "--\n" "\n" "Return a copy with all occurrences of substring old replaced by new.\n" @@ -798,25 +798,56 @@ PyDoc_STRVAR(bytes_replace__doc__, " Maximum number of occurrences to replace.\n" " -1 (the default value) means replace all occurrences.\n" "\n" -"If the optional argument count is given, only the first count occurrences are\n" -"replaced."); +"If count is given, only the first count occurrences are replaced.\n" +"If count is not specified or -1, then all occurrences are replaced."); #define BYTES_REPLACE_METHODDEF \ - {"replace", _PyCFunction_CAST(bytes_replace), METH_FASTCALL, bytes_replace__doc__}, + {"replace", _PyCFunction_CAST(bytes_replace), METH_FASTCALL|METH_KEYWORDS, bytes_replace__doc__}, static PyObject * bytes_replace_impl(PyBytesObject *self, Py_buffer *old, Py_buffer *new, Py_ssize_t count); static PyObject * -bytes_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +bytes_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(count), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "", "count", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "replace", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; Py_buffer old = {NULL, NULL}; Py_buffer new = {NULL, NULL}; Py_ssize_t count = -1; - if (!_PyArg_CheckPositional("replace", nargs, 2, 3)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { goto exit; } if (PyObject_GetBuffer(args[0], &old, PyBUF_SIMPLE) != 0) { @@ -825,8 +856,8 @@ bytes_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[1], &new, PyBUF_SIMPLE) != 0) { goto exit; } - if (nargs < 3) { - goto skip_optional; + if (!noptargs) { + goto skip_optional_pos; } { Py_ssize_t ival = -1; @@ -840,7 +871,7 @@ bytes_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs) } count = ival; } -skip_optional: +skip_optional_pos: return_value = bytes_replace_impl((PyBytesObject *)self, &old, &new, count); exit: @@ -1254,7 +1285,7 @@ PyDoc_STRVAR(bytes_hex__doc__, {"hex", _PyCFunction_CAST(bytes_hex), METH_FASTCALL|METH_KEYWORDS, bytes_hex__doc__}, static PyObject * -bytes_hex_impl(PyBytesObject *self, PyObject *sep, int bytes_per_sep); +bytes_hex_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t bytes_per_sep); static PyObject * bytes_hex(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -1290,7 +1321,7 @@ bytes_hex(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *sep = NULL; - int bytes_per_sep = 1; + Py_ssize_t bytes_per_sep = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -1306,9 +1337,17 @@ bytes_hex(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn goto skip_optional_pos; } } - bytes_per_sep = PyLong_AsInt(args[1]); - if (bytes_per_sep == -1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + bytes_per_sep = ival; } skip_optional_pos: return_value = bytes_hex_impl((PyBytesObject *)self, sep, bytes_per_sep); @@ -1411,4 +1450,4 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=08b9507244f73638 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b252801ff04a89b3 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index 28cfd1a22080c9..d97c626532c803 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(memoryview__doc__, @@ -366,7 +367,7 @@ PyDoc_STRVAR(memoryview_hex__doc__, static PyObject * memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, - int bytes_per_sep); + Py_ssize_t bytes_per_sep); static PyObject * memoryview_hex(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -402,7 +403,7 @@ memoryview_hex(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *sep = NULL; - int bytes_per_sep = 1; + Py_ssize_t bytes_per_sep = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -418,9 +419,17 @@ memoryview_hex(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_pos; } } - bytes_per_sep = PyLong_AsInt(args[1]); - if (bytes_per_sep == -1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + bytes_per_sep = ival; } skip_optional_pos: return_value = memoryview_hex_impl((PyMemoryViewObject *)self, sep, bytes_per_sep); @@ -496,4 +505,4 @@ memoryview_index(PyObject *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=154f4c04263ccb24 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=348b6ddb98a1f412 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index 1819fbaea220a3..4b53e24fb7d649 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -918,8 +918,8 @@ PyDoc_STRVAR(unicode_replace__doc__, " Maximum number of occurrences to replace.\n" " -1 (the default value) means replace all occurrences.\n" "\n" -"If the optional argument count is given, only the first count occurrences are\n" -"replaced."); +"If count is given, only the first count occurrences are replaced.\n" +"If count is not specified or -1, then all occurrences are replaced."); #define UNICODE_REPLACE_METHODDEF \ {"replace", _PyCFunction_CAST(unicode_replace), METH_FASTCALL|METH_KEYWORDS, unicode_replace__doc__}, @@ -1908,4 +1908,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=238917fe66120bde input=a9049054013a1b77]*/ +/*[clinic end generated code: output=13eaf65699ea9fc9 input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index fbf0985e9050dd..ad1116890e23ce 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -501,7 +501,7 @@ _PyCode_Validate(struct _PyCodeConstructor *con) } extern void -_PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, int enable_counters); +_PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, int enable_counters, int flags); #ifdef Py_GIL_DISABLED static _PyCodeArray * _PyCodeArray_New(Py_ssize_t size); @@ -579,11 +579,12 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) entry_point++; } co->_co_firsttraceable = entry_point; + #ifdef Py_GIL_DISABLED int enable_counters = interp->config.tlbc_enabled && interp->opt_config.specialization_enabled; - _PyCode_Quicken(_PyCode_CODE(co), Py_SIZE(co), enable_counters); + _PyCode_Quicken(_PyCode_CODE(co), Py_SIZE(co), enable_counters, co->co_flags); #else - _PyCode_Quicken(_PyCode_CODE(co), Py_SIZE(co), interp->opt_config.specialization_enabled); + _PyCode_Quicken(_PyCode_CODE(co), Py_SIZE(co), interp->opt_config.specialization_enabled, co->co_flags); #endif notify_code_watchers(PY_CODE_EVENT_CREATE, co); return 0; @@ -2606,7 +2607,7 @@ code_richcompare(PyObject *self, PyObject *other, int op) cp = (PyCodeObject *)other; eq = PyObject_RichCompareBool(co->co_name, cp->co_name, Py_EQ); - if (!eq) goto unequal; + if (eq <= 0) goto unequal; eq = co->co_argcount == cp->co_argcount; if (!eq) goto unequal; eq = co->co_posonlyargcount == cp->co_posonlyargcount; @@ -3051,7 +3052,7 @@ _PyCode_ConstantKey(PyObject *op) else if (PyBool_Check(op) || PyBytes_CheckExact(op)) { /* Make booleans different from integers 0 and 1. * Avoid BytesWarning from comparing bytes with strings. */ - key = PyTuple_Pack(2, Py_TYPE(op), op); + key = _PyTuple_FromPair((PyObject *)Py_TYPE(op), op); } else if (PyFloat_CheckExact(op)) { double d = PyFloat_AS_DOUBLE(op); @@ -3061,7 +3062,7 @@ _PyCode_ConstantKey(PyObject *op) if (d == 0.0 && copysign(1.0, d) < 0.0) key = PyTuple_Pack(3, Py_TYPE(op), op, Py_None); else - key = PyTuple_Pack(2, Py_TYPE(op), op); + key = _PyTuple_FromPair((PyObject *)Py_TYPE(op), op); } else if (PyComplex_CheckExact(op)) { Py_complex z; @@ -3085,7 +3086,7 @@ _PyCode_ConstantKey(PyObject *op) key = PyTuple_Pack(3, Py_TYPE(op), op, Py_None); } else { - key = PyTuple_Pack(2, Py_TYPE(op), op); + key = _PyTuple_FromPair((PyObject *)Py_TYPE(op), op); } } else if (PyTuple_CheckExact(op)) { @@ -3110,7 +3111,7 @@ _PyCode_ConstantKey(PyObject *op) PyTuple_SET_ITEM(tuple, i, item_key); } - key = PyTuple_Pack(2, tuple, op); + key = _PyTuple_FromPair(tuple, op); Py_DECREF(tuple); } else if (PyFrozenSet_CheckExact(op)) { @@ -3144,7 +3145,7 @@ _PyCode_ConstantKey(PyObject *op) if (set == NULL) return NULL; - key = PyTuple_Pack(2, set, op); + key = _PyTuple_FromPair(set, op); Py_DECREF(set); return key; } @@ -3175,7 +3176,7 @@ _PyCode_ConstantKey(PyObject *op) goto slice_exit; } - key = PyTuple_Pack(2, slice_key, op); + key = _PyTuple_FromPair(slice_key, op); Py_DECREF(slice_key); slice_exit: Py_XDECREF(start_key); @@ -3189,7 +3190,7 @@ _PyCode_ConstantKey(PyObject *op) if (obj_id == NULL) return NULL; - key = PyTuple_Pack(2, obj_id, op); + key = _PyTuple_FromPair(obj_id, op); Py_DECREF(obj_id); } return key; @@ -3460,7 +3461,7 @@ copy_code(PyInterpreterState *interp, _Py_CODEUNIT *dst, PyCodeObject *co) for (int i = 0; i < code_len; i += _PyInstruction_GetLength(co, i)) { dst[i] = deopt_code_unit(co, i); } - _PyCode_Quicken(dst, code_len, interp->opt_config.specialization_enabled); + _PyCode_Quicken(dst, code_len, interp->opt_config.specialization_enabled, co->co_flags); } static Py_ssize_t diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 5ac4fbd812924c..a5926616eeb3cb 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -150,7 +150,7 @@ method_get(PyObject *self, PyObject *obj, PyObject *type) } else { PyErr_Format(PyExc_TypeError, "descriptor '%V' needs a type, not '%s', as arg 2", - descr_name((PyDescrObject *)descr), + descr_name((PyDescrObject *)descr), "?", Py_TYPE(type)->tp_name); return NULL; } @@ -1610,7 +1610,7 @@ property_set_name(PyObject *self, PyObject *args) { if (PyTuple_GET_SIZE(args) != 2) { PyErr_Format( PyExc_TypeError, - "__set_name__() takes 2 positional arguments but %d were given", + "__set_name__() takes 2 positional arguments but %zd were given", PyTuple_GET_SIZE(args)); return NULL; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 08e40bf84c42fa..67bc4319e0bae2 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -139,7 +139,7 @@ As a consequence of this, split keys have a maximum size of 16. static PyObject* frozendict_new(PyTypeObject *type, PyObject *args, PyObject *kwds); static PyObject* dict_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static int dict_merge(PyObject *a, PyObject *b, int override); +static int dict_merge(PyObject *a, PyObject *b, int override, PyObject **dupkey); static int dict_contains(PyObject *op, PyObject *key); static int dict_merge_from_seq2(PyObject *d, PyObject *seq2, int override); @@ -2727,13 +2727,23 @@ _PyDict_LoadBuiltinsFromGlobals(PyObject *globals) /* Consumes references to key and value */ static int -setitem_take2_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) +setitem_take2_lock_held_known_hash(PyDictObject *mp, PyObject *key, PyObject *value, Py_hash_t hash) { assert(PyAnyDict_Check(mp)); assert(can_modify_dict(mp)); assert(key); assert(value); + if (mp->ma_keys == Py_EMPTY_KEYS) { + return insert_to_emptydict(mp, key, hash, value); + } + /* insertdict() handles any resizing that might be necessary */ + return insertdict(mp, key, hash, value); +} + +static int +setitem_take2_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) +{ Py_hash_t hash = _PyObject_HashFast(key); if (hash == -1) { dict_unhashable_type((PyObject*)mp, key); @@ -2742,11 +2752,7 @@ setitem_take2_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) return -1; } - if (mp->ma_keys == Py_EMPTY_KEYS) { - return insert_to_emptydict(mp, key, hash, value); - } - /* insertdict() handles any resizing that might be necessary */ - return insertdict(mp, key, hash, value); + return setitem_take2_lock_held_known_hash(mp, key, value, hash); } int @@ -2759,6 +2765,16 @@ _PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) return res; } +int +_PyDict_SetItem_Take2_KnownHash(PyDictObject *mp, PyObject *key, PyObject *value, Py_hash_t hash) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(mp); + res = setitem_take2_lock_held_known_hash(mp, key, value, hash); + Py_END_CRITICAL_SECTION(); + return res; +} + /* CAUTION: PyDict_SetItem() must guarantee that it won't resize the * dictionary if it's merely replacing the value for an existing key. * This means that it's safe to loop over a dictionary with PyDict_Next() @@ -3391,7 +3407,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) Py_DECREF(d); return NULL; } - if (dict_merge(copy, d, 1) < 0) { + if (dict_merge(copy, d, 1, NULL) < 0) { Py_DECREF(d); Py_DECREF(copy); return NULL; @@ -3887,14 +3903,14 @@ static int dict_update_arg(PyObject *self, PyObject *arg) { if (PyAnyDict_CheckExact(arg)) { - return dict_merge(self, arg, 1); + return dict_merge(self, arg, 1, NULL); } int has_keys = PyObject_HasAttrWithError(arg, &_Py_ID(keys)); if (has_keys < 0) { return -1; } if (has_keys) { - return dict_merge(self, arg, 1); + return dict_merge(self, arg, 1, NULL); } return dict_merge_from_seq2(self, arg, 1); } @@ -3915,7 +3931,7 @@ dict_update_common(PyObject *self, PyObject *args, PyObject *kwds, if (result == 0 && kwds != NULL) { if (PyArg_ValidateKeywordArguments(kwds)) - result = dict_merge(self, kwds, 1); + result = dict_merge(self, kwds, 1, NULL); else result = -1; } @@ -4059,7 +4075,7 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) } static int -dict_dict_merge(PyDictObject *mp, PyDictObject *other, int override) +dict_dict_merge(PyDictObject *mp, PyDictObject *other, int override, PyObject **dupkey) { assert(can_modify_dict(mp)); ASSERT_DICT_LOCKED(other); @@ -4068,10 +4084,10 @@ dict_dict_merge(PyDictObject *mp, PyDictObject *other, int override) /* a.update(a) or a.update({}); nothing to do */ return 0; if (mp->ma_used == 0) { - /* Since the target dict is empty, PyDict_GetItem() - * always returns NULL. Setting override to 1 - * skips the unnecessary test. - */ + /* Since the target dict is empty, _PyDict_Contains_KnownHash() + * always returns 0. Setting override to 1 + * skips the unnecessary test. + */ override = 1; PyDictKeysObject *okeys = other->ma_keys; @@ -4131,11 +4147,10 @@ dict_dict_merge(PyDictObject *mp, PyDictObject *other, int override) err = insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value)); } else if (err > 0) { - if (override != 0) { - _PyErr_SetKeyError(key); + if (dupkey != NULL) { + *dupkey = key; Py_DECREF(value); - Py_DECREF(key); - return -1; + return -2; } err = 0; } @@ -4155,7 +4170,7 @@ dict_dict_merge(PyDictObject *mp, PyDictObject *other, int override) } static int -dict_merge(PyObject *a, PyObject *b, int override) +dict_merge(PyObject *a, PyObject *b, int override, PyObject **dupkey) { assert(a != NULL); assert(b != NULL); @@ -4167,7 +4182,7 @@ dict_merge(PyObject *a, PyObject *b, int override) PyDictObject *other = (PyDictObject*)b; int res; Py_BEGIN_CRITICAL_SECTION2(a, b); - res = dict_dict_merge((PyDictObject *)a, other, override); + res = dict_dict_merge((PyDictObject *)a, other, override, dupkey); ASSERT_CONSISTENT(a); Py_END_CRITICAL_SECTION2(); return res; @@ -4202,15 +4217,18 @@ dict_merge(PyObject *a, PyObject *b, int override) status = dict_contains(a, key); if (status != 0) { if (status > 0) { - if (override == 0) { + if (dupkey == NULL) { Py_DECREF(key); continue; } - _PyErr_SetKeyError(key); + *dupkey = key; + res = -2; + } + else { + Py_DECREF(key); + res = -1; } - Py_DECREF(key); Py_DECREF(iter); - res = -1; goto slow_exit; } } @@ -4246,7 +4264,7 @@ dict_merge(PyObject *a, PyObject *b, int override) } static int -dict_merge_api(PyObject *a, PyObject *b, int override) +dict_merge_api(PyObject *a, PyObject *b, int override, PyObject **dupkey) { /* We accept for the argument either a concrete dictionary object, * or an abstract "mapping" object. For the former, we can do @@ -4262,26 +4280,26 @@ dict_merge_api(PyObject *a, PyObject *b, int override) } return -1; } - return dict_merge(a, b, override); + return dict_merge(a, b, override, dupkey); } int PyDict_Update(PyObject *a, PyObject *b) { - return dict_merge_api(a, b, 1); + return dict_merge_api(a, b, 1, NULL); } int PyDict_Merge(PyObject *a, PyObject *b, int override) { /* XXX Deprecate override not in (0, 1). */ - return dict_merge_api(a, b, override != 0); + return dict_merge_api(a, b, override != 0, NULL); } int -_PyDict_MergeEx(PyObject *a, PyObject *b, int override) +_PyDict_MergeUniq(PyObject *a, PyObject *b, PyObject **dupkey) { - return dict_merge_api(a, b, override); + return dict_merge_api(a, b, 2, dupkey); } /*[clinic input] @@ -4421,7 +4439,7 @@ copy_lock_held(PyObject *o, int as_frozendict) } if (copy == NULL) return NULL; - if (dict_merge(copy, o, 1) == 0) + if (dict_merge(copy, o, 1, NULL) == 0) return copy; Py_DECREF(copy); return NULL; @@ -5454,7 +5472,7 @@ dictiter_new(PyDictObject *dict, PyTypeObject *itertype) } if (itertype == &PyDictIterItem_Type || itertype == &PyDictRevIterItem_Type) { - di->di_result = PyTuple_Pack(2, Py_None, Py_None); + di->di_result = _PyTuple_FromPairSteal(Py_None, Py_None); if (di->di_result == NULL) { Py_DECREF(di); return NULL; @@ -6020,14 +6038,7 @@ dictiter_iternextitem(PyObject *self) _PyTuple_Recycle(result); } else { - result = PyTuple_New(2); - if (result == NULL) { - Py_DECREF(key); - Py_DECREF(value); - return NULL; - } - PyTuple_SET_ITEM(result, 0, key); - PyTuple_SET_ITEM(result, 1, value); + result = _PyTuple_FromPairSteal(key, value); } return result; } @@ -6146,12 +6157,7 @@ dictreviter_iter_lock_held(PyDictObject *d, PyObject *self) _PyTuple_Recycle(result); } else { - result = PyTuple_New(2); - if (result == NULL) { - return NULL; - } - PyTuple_SET_ITEM(result, 0, Py_NewRef(key)); - PyTuple_SET_ITEM(result, 1, Py_NewRef(value)); + result = _PyTuple_FromPair(key, value); } return result; } @@ -6644,18 +6650,22 @@ dictitems_xor_lock_held(PyObject *d1, PyObject *d2) else { Py_INCREF(val1); to_delete = PyObject_RichCompareBool(val1, val2, Py_EQ); + Py_CLEAR(val1); if (to_delete < 0) { goto error; } } if (to_delete) { + Py_CLEAR(val2); if (_PyDict_DelItem_KnownHash(temp_dict, key, hash) < 0) { goto error; } + Py_CLEAR(key); } else { - PyObject *pair = PyTuple_Pack(2, key, val2); + PyObject *pair = _PyTuple_FromPairSteal(key, val2); + key = val2 = NULL; if (pair == NULL) { goto error; } @@ -6665,11 +6675,7 @@ dictitems_xor_lock_held(PyObject *d1, PyObject *d2) } Py_DECREF(pair); } - Py_DECREF(key); - Py_XDECREF(val1); - Py_DECREF(val2); } - key = val1 = val2 = NULL; PyObject *remaining_pairs = PyObject_CallMethodNoArgs( temp_dict, &_Py_ID(items)); diff --git a/Objects/enumobject.c b/Objects/enumobject.c index 70e7cce6aba008..364d508dd01822 100644 --- a/Objects/enumobject.c +++ b/Objects/enumobject.c @@ -78,7 +78,7 @@ enum_new_impl(PyTypeObject *type, PyObject *iterable, PyObject *start) Py_DECREF(en); return NULL; } - en->en_result = PyTuple_Pack(2, Py_None, Py_None); + en->en_result = _PyTuple_FromPairSteal(Py_None, Py_None); if (en->en_result == NULL) { Py_DECREF(en); return NULL; @@ -148,7 +148,7 @@ enumerate_vectorcall(PyObject *type, PyObject *const *args, } PyErr_Format(PyExc_TypeError, - "enumerate() takes at most 2 arguments (%d given)", nargs + nkwargs); + "enumerate() takes at most 2 arguments (%zd given)", nargs + nkwargs); return NULL; } @@ -226,15 +226,7 @@ enum_next_long(enumobject *en, PyObject* next_item) _PyTuple_Recycle(result); return result; } - result = PyTuple_New(2); - if (result == NULL) { - Py_DECREF(next_index); - Py_DECREF(next_item); - return NULL; - } - PyTuple_SET_ITEM(result, 0, next_index); - PyTuple_SET_ITEM(result, 1, next_item); - return result; + return _PyTuple_FromPairSteal(next_index, next_item); } static PyObject * @@ -276,15 +268,7 @@ enum_next(PyObject *op) _PyTuple_Recycle(result); return result; } - result = PyTuple_New(2); - if (result == NULL) { - Py_DECREF(next_index); - Py_DECREF(next_item); - return NULL; - } - PyTuple_SET_ITEM(result, 0, next_index); - PyTuple_SET_ITEM(result, 1, next_item); - return result; + return _PyTuple_FromPairSteal(next_index, next_item); } static PyObject * diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 32b0fcec6c4542..5e5e87cd6d7559 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -935,7 +935,7 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (!PyExceptionInstance_Check(exc)) { PyErr_Format( PyExc_ValueError, - "Item %d of second argument (exceptions) is not an exception", + "Item %zd of second argument (exceptions) is not an exception", i); goto error; } @@ -1714,7 +1714,7 @@ PyUnstable_Exc_PrepReraiseStar(PyObject *orig, PyObject *excs) PyObject *exc = PyList_GET_ITEM(excs, i); if (exc == NULL || !(PyExceptionInstance_Check(exc) || Py_IsNone(exc))) { PyErr_Format(PyExc_TypeError, - "item %d of excs is not an exception", i); + "item %zd of excs is not an exception", i); return NULL; } } @@ -2802,23 +2802,25 @@ SyntaxError_init(PyObject *op, PyObject *args, PyObject *kwds) return -1; } - self->end_lineno = NULL; - self->end_offset = NULL; + PyObject *filename, *lineno, *offset, *text; + PyObject *end_lineno = NULL; + PyObject *end_offset = NULL; + PyObject *metadata = NULL; if (!PyArg_ParseTuple(info, "OOOO|OOO", - &self->filename, &self->lineno, - &self->offset, &self->text, - &self->end_lineno, &self->end_offset, &self->metadata)) { + &filename, &lineno, + &offset, &text, + &end_lineno, &end_offset, &metadata)) { Py_DECREF(info); return -1; } - Py_INCREF(self->filename); - Py_INCREF(self->lineno); - Py_INCREF(self->offset); - Py_INCREF(self->text); - Py_XINCREF(self->end_lineno); - Py_XINCREF(self->end_offset); - Py_XINCREF(self->metadata); + Py_XSETREF(self->filename, Py_NewRef(filename)); + Py_XSETREF(self->lineno, Py_NewRef(lineno)); + Py_XSETREF(self->offset, Py_NewRef(offset)); + Py_XSETREF(self->text, Py_NewRef(text)); + Py_XSETREF(self->end_lineno, Py_XNewRef(end_lineno)); + Py_XSETREF(self->end_offset, Py_XNewRef(end_offset)); + Py_XSETREF(self->metadata, Py_XNewRef(metadata)); Py_DECREF(info); if (self->end_lineno != NULL && self->end_offset == NULL) { diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 18871a4f3c51a9..d91468dddded9b 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -16,6 +16,7 @@ #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_stackref.h" // PyStackRef_AsPyObjectBorrow() #include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() +#include "pycore_tuple.h" // _PyTuple_FromPair #include // DBL_MAX #include // strtol() @@ -1539,8 +1540,9 @@ float_as_integer_ratio_impl(PyObject *self) if (denominator == NULL) goto error; } + Py_DECREF(py_exponent); - result_pair = PyTuple_Pack(2, numerator, denominator); + return _PyTuple_FromPairSteal(numerator, denominator); error: Py_XDECREF(py_exponent); diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 8911de6f2bfc5b..5ae85c5bca61b9 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -13,6 +13,7 @@ #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_opcode_metadata.h" // _PyOpcode_Caches #include "pycore_optimizer.h" // _Py_Executors_InvalidateDependency() +#include "pycore_tuple.h" // _PyTuple_FromPair #include "pycore_unicodeobject.h" // _PyUnicode_Equal() #include "frameobject.h" // PyFrameLocalsProxyObject @@ -630,22 +631,16 @@ framelocalsproxy_items(PyObject *self, PyObject *Py_UNUSED(ignored)) PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); if (value) { - PyObject *pair = PyTuple_Pack(2, name, value); + PyObject *pair = _PyTuple_FromPairSteal(Py_NewRef(name), value); if (pair == NULL) { - Py_DECREF(items); - Py_DECREF(value); - return NULL; - } - - if (PyList_Append(items, pair) < 0) { - Py_DECREF(items); - Py_DECREF(pair); - Py_DECREF(value); - return NULL; + goto error; } + int rc = PyList_Append(items, pair); Py_DECREF(pair); - Py_DECREF(value); + if (rc < 0) { + goto error; + } } } @@ -655,23 +650,24 @@ framelocalsproxy_items(PyObject *self, PyObject *Py_UNUSED(ignored)) PyObject *key = NULL; PyObject *value = NULL; while (PyDict_Next(frame->f_extra_locals, &j, &key, &value)) { - PyObject *pair = PyTuple_Pack(2, key, value); + PyObject *pair = _PyTuple_FromPair(key, value); if (pair == NULL) { - Py_DECREF(items); - return NULL; - } - - if (PyList_Append(items, pair) < 0) { - Py_DECREF(items); - Py_DECREF(pair); - return NULL; + goto error; } + int rc = PyList_Append(items, pair); Py_DECREF(pair); + if (rc < 0) { + goto error; + } } } return items; + +error: + Py_DECREF(items); + return NULL; } static Py_ssize_t diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 585c7b9a85412c..d47c78b933b702 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -657,7 +657,7 @@ func_set_code(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) if (nclosure != nfree) { PyErr_Format(PyExc_ValueError, "%U() requires a code object with %zd free vars," - " not %zd", + " not %d", op->func_name, nclosure, nfree); return -1; @@ -1044,7 +1044,7 @@ func_new_impl(PyTypeObject *type, PyCodeObject *code, PyObject *globals, nclosure = closure == Py_None ? 0 : PyTuple_GET_SIZE(closure); if (code->co_nfreevars != nclosure) return PyErr_Format(PyExc_ValueError, - "%U requires closure of length %zd, not %zd", + "%U requires closure of length %d, not %zd", code->co_name, code->co_nfreevars, nclosure); if (nclosure) { Py_ssize_t i; diff --git a/Objects/genobject.c b/Objects/genobject.c index 9dece8a7700cab..2895833b4ff933 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -490,7 +490,7 @@ gen_close(PyObject *self, PyObject *args) int err = 0; _PyInterpreterFrame *frame = &gen->gi_iframe; if (frame_state == FRAME_SUSPENDED_YIELD_FROM) { - PyObject *yf = PyStackRef_AsPyObjectNew(_PyFrame_StackPeek(frame)); + PyObject *yf = PyStackRef_AsPyObjectNew(_PyFrame_StackPeek(frame, 2)); err = gen_close_iter(yf); Py_DECREF(yf); } @@ -649,7 +649,7 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, if (frame_state == FRAME_SUSPENDED_YIELD_FROM) { _PyInterpreterFrame *frame = &gen->gi_iframe; - PyObject *yf = PyStackRef_AsPyObjectNew(_PyFrame_StackPeek(frame)); + PyObject *yf = PyStackRef_AsPyObjectNew(_PyFrame_StackPeek(frame, 2)); PyObject *ret; int err; if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit) && @@ -898,7 +898,7 @@ gen_getyieldfrom(PyObject *self, void *Py_UNUSED(ignored)) } } while (!_Py_GEN_TRY_SET_FRAME_STATE(gen, frame_state, FRAME_SUSPENDED_YIELD_FROM_LOCKED)); - PyObject *result = PyStackRef_AsPyObjectNew(_PyFrame_StackPeek(&gen->gi_iframe)); + PyObject *result = PyStackRef_AsPyObjectNew(_PyFrame_StackPeek(&gen->gi_iframe, 2)); _Py_atomic_store_int8_release(&gen->gi_frame_state, FRAME_SUSPENDED_YIELD_FROM); return result; #else @@ -906,7 +906,7 @@ gen_getyieldfrom(PyObject *self, void *Py_UNUSED(ignored)) if (frame_state != FRAME_SUSPENDED_YIELD_FROM) { Py_RETURN_NONE; } - return PyStackRef_AsPyObjectNew(_PyFrame_StackPeek(&gen->gi_iframe)); + return PyStackRef_AsPyObjectNew(_PyFrame_StackPeek(&gen->gi_iframe, 2)); #endif } diff --git a/Objects/listobject.c b/Objects/listobject.c index 654b8130e70840..5c9fd55bab1b22 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -1437,9 +1437,9 @@ list_extend_dictitems(PyListObject *self, PyDictObject *dict) PyObject **dest = self->ob_item + m; Py_ssize_t pos = 0; Py_ssize_t i = 0; - PyObject *key_value[2]; - while (_PyDict_Next((PyObject *)dict, &pos, &key_value[0], &key_value[1], NULL)) { - PyObject *item = PyTuple_FromArray(key_value, 2); + PyObject *key, *value; + while (_PyDict_Next((PyObject *)dict, &pos, &key, &value, NULL)) { + PyObject *item = _PyTuple_FromPair(key, value); if (item == NULL) { Py_SET_SIZE(self, m + i); return -1; diff --git a/Objects/longobject.c b/Objects/longobject.c index 7ce5d0535b884e..549cf0b8f12b4e 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -12,6 +12,7 @@ #include "pycore_runtime.h" // _PY_NSMALLPOSINTS #include "pycore_stackref.h" #include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() +#include "pycore_tuple.h" // _PyTuple_FromPairSteal #include "pycore_unicodeobject.h" // _PyUnicode_Equal() #include // DBL_MANT_DIG @@ -25,7 +26,7 @@ class int "PyObject *" "&PyLong_Type" #define medium_value(x) ((stwodigits)_PyLong_CompactValue(x)) -#define IS_SMALL_INT(ival) (-_PY_NSMALLNEGINTS <= (ival) && (ival) < _PY_NSMALLPOSINTS) +#define IS_SMALL_INT(ival) _PY_IS_SMALL_INT(ival) #define IS_SMALL_UINT(ival) ((ival) < _PY_NSMALLPOSINTS) #define _MAX_STR_DIGITS_ERROR_FMT_TO_INT "Exceeds the limit (%d digits) for integer string conversion: value has %zd digits; use sys.set_int_max_str_digits() to increase the limit" @@ -184,11 +185,14 @@ long_alloc(Py_ssize_t size) return NULL; } _PyObject_Init((PyObject*)result, &PyLong_Type); + _PyLong_InitTag(result); } _PyLong_SetSignAndDigitCount(result, size != 0, size); - /* The digit has to be initialized explicitly to avoid - * use-of-uninitialized-value. */ - result->long_value.ob_digit[0] = 0; +#ifdef Py_DEBUG + // gh-147988: Fill digits with an invalid pattern to catch usage + // of uninitialized digits. + memset(result->long_value.ob_digit, 0xFF, ndigits * sizeof(digit)); +#endif return result; } @@ -257,6 +261,7 @@ _PyLong_FromMedium(sdigit x) return NULL; } _PyObject_Init((PyObject*)v, &PyLong_Type); + _PyLong_InitTag(v); } digit abs_x = x < 0 ? -x : x; _PyLong_SetSignAndDigitCount(v, x<0?-1:1, 1); @@ -336,6 +341,7 @@ medium_from_stwodigits(stwodigits x) return PyStackRef_NULL; } _PyObject_Init((PyObject*)v, &PyLong_Type); + _PyLong_InitTag(v); } digit abs_x = x < 0 ? (digit)(-x) : (digit)x; _PyLong_SetSignAndDigitCount(v, x<0?-1:1, 1); @@ -1093,6 +1099,7 @@ _PyLong_FromByteArray(const unsigned char* bytes, size_t n, int sign = is_signed ? -1: 1; if (idigit == 0) { sign = 0; + v->long_value.ob_digit[0] = 0; } _PyLong_SetSignAndDigitCount(v, sign, idigit); return (PyObject *)maybe_small_long(long_normalize(v)); @@ -2851,6 +2858,7 @@ long_from_non_binary_base(const char *start, const char *end, Py_ssize_t digits, *res = NULL; return 0; } + z->long_value.ob_digit[0] = 0; _PyLong_SetSignAndDigitCount(z, 0, 0); /* `convwidth` consecutive input digits are treated as a single @@ -3118,11 +3126,11 @@ PyLong_FromString(const char *str, char **pend, int base) } /* Set sign and normalize */ - if (sign < 0) { - _PyLong_FlipSign(z); - } long_normalize(z); z = maybe_small_long(z); + if (sign < 0) { + _PyLong_Negate(&z); + } if (pend != NULL) { *pend = (char *)str; @@ -3364,6 +3372,7 @@ x_divrem(PyLongObject *v1, PyLongObject *w1, PyLongObject **prem) *prem = NULL; return NULL; } + a->long_value.ob_digit[0] = 0; v0 = v->long_value.ob_digit; w0 = w->long_value.ob_digit; wm1 = w0[size_w-1]; @@ -3622,21 +3631,11 @@ long_richcompare(PyObject *self, PyObject *other, int op) Py_RETURN_RICHCOMPARE(result, 0, op); } -static inline int -/// Return 1 if the object is one of the immortal small ints -_long_is_small_int(PyObject *op) -{ - PyLongObject *long_object = (PyLongObject *)op; - int is_small_int = (long_object->long_value.lv_tag & IMMORTALITY_BIT_MASK) != 0; - assert((!is_small_int) || PyLong_CheckExact(op)); - return is_small_int; -} - void _PyLong_ExactDealloc(PyObject *self) { assert(PyLong_CheckExact(self)); - if (_long_is_small_int(self)) { + if (_PyLong_IsSmallInt((PyLongObject *)self)) { // See PEP 683, section Accidental De-Immortalizing for details _Py_SetImmortal(self); return; @@ -3651,7 +3650,7 @@ _PyLong_ExactDealloc(PyObject *self) static void long_dealloc(PyObject *self) { - if (_long_is_small_int(self)) { + if (_PyLong_IsSmallInt((PyLongObject *)self)) { /* This should never get called, but we also don't want to SEGV if * we accidentally decref small Ints out of existence. Instead, * since small Ints are immortal, re-set the reference count. @@ -4150,10 +4149,6 @@ k_mul(PyLongObject *a, PyLongObject *b) /* 1. Allocate result space. */ ret = long_alloc(asize + bsize); if (ret == NULL) goto fail; -#ifdef Py_DEBUG - /* Fill with trash, to catch reference to uninitialized digits. */ - memset(ret->long_value.ob_digit, 0xDF, _PyLong_DigitCount(ret) * sizeof(digit)); -#endif /* 2. t1 <- ah*bh, and copy into high digits of result. */ if ((t1 = k_mul(ah, bh)) == NULL) goto fail; @@ -4878,23 +4873,12 @@ static PyObject * long_divmod(PyObject *a, PyObject *b) { PyLongObject *div, *mod; - PyObject *z; - CHECK_BINOP(a, b); if (l_divmod((PyLongObject*)a, (PyLongObject*)b, &div, &mod) < 0) { return NULL; } - z = PyTuple_New(2); - if (z != NULL) { - PyTuple_SET_ITEM(z, 0, (PyObject *) div); - PyTuple_SET_ITEM(z, 1, (PyObject *) mod); - } - else { - Py_DECREF(div); - Py_DECREF(mod); - } - return z; + return _PyTuple_FromPairSteal((PyObject *)div, (PyObject *)mod); } @@ -5653,6 +5637,12 @@ long_bitwise(PyLongObject *a, Py_UNREACHABLE(); } + if ((size_z + negz) == 0) { + Py_XDECREF(new_a); + Py_XDECREF(new_b); + return get_small_int(0); + } + /* We allow an extra digit if z is negative, to make sure that the final two's complement of z doesn't overflow. */ z = long_alloc(size_z + negz); @@ -6031,29 +6021,34 @@ static PyObject * long_subtype_new(PyTypeObject *type, PyObject *x, PyObject *obase) { PyLongObject *tmp, *newobj; - Py_ssize_t i, n; + Py_ssize_t size, ndigits; + int sign; assert(PyType_IsSubtype(type, &PyLong_Type)); tmp = (PyLongObject *)long_new_impl(&PyLong_Type, x, obase); if (tmp == NULL) return NULL; assert(PyLong_Check(tmp)); - n = _PyLong_DigitCount(tmp); + size = _PyLong_DigitCount(tmp); /* Fast operations for single digit integers (including zero) * assume that there is always at least one digit present. */ - if (n == 0) { - n = 1; - } - newobj = (PyLongObject *)type->tp_alloc(type, n); + ndigits = size ? size : 1; + newobj = (PyLongObject *)type->tp_alloc(type, ndigits); if (newobj == NULL) { Py_DECREF(tmp); return NULL; } assert(PyLong_Check(newobj)); - newobj->long_value.lv_tag = tmp->long_value.lv_tag & ~IMMORTALITY_BIT_MASK; - for (i = 0; i < n; i++) { - newobj->long_value.ob_digit[i] = tmp->long_value.ob_digit[i]; + if (_PyLong_IsCompact(tmp)) { + sign = _PyLong_CompactSign(tmp); + } + else { + sign = _PyLong_NonCompactSign(tmp); } + _PyLong_InitTag(newobj); + _PyLong_SetSignAndDigitCount(newobj, sign, size); + memcpy(newobj->long_value.ob_digit, tmp->long_value.ob_digit, + ndigits * sizeof(digit)); Py_DECREF(tmp); return (PyObject *)newobj; } @@ -6118,7 +6113,7 @@ PyObject * _PyLong_DivmodNear(PyObject *a, PyObject *b) { PyLongObject *quo = NULL, *rem = NULL; - PyObject *twice_rem, *result, *temp; + PyObject *twice_rem, *temp; int quo_is_odd, quo_is_neg; Py_ssize_t cmp; @@ -6184,14 +6179,7 @@ _PyLong_DivmodNear(PyObject *a, PyObject *b) goto error; } - result = PyTuple_New(2); - if (result == NULL) - goto error; - - /* PyTuple_SET_ITEM steals references */ - PyTuple_SET_ITEM(result, 0, (PyObject *)quo); - PyTuple_SET_ITEM(result, 1, (PyObject *)rem); - return result; + return _PyTuple_FromPairSteal((PyObject *)quo, (PyObject *)rem); error: Py_XDECREF(quo); @@ -6368,14 +6356,11 @@ static PyObject * int_as_integer_ratio_impl(PyObject *self) /*[clinic end generated code: output=e60803ae1cc8621a input=384ff1766634bec2]*/ { - PyObject *ratio_tuple; PyObject *numerator = long_long(self); if (numerator == NULL) { return NULL; } - ratio_tuple = PyTuple_Pack(2, numerator, _PyLong_GetOne()); - Py_DECREF(numerator); - return ratio_tuple; + return _PyTuple_FromPairSteal(numerator, _PyLong_GetOne()); } /*[clinic input] @@ -6981,6 +6966,28 @@ PyLongWriter_Finish(PyLongWriter *writer) PyLongObject *obj = (PyLongObject *)writer; assert(Py_REFCNT(obj) == 1); +#ifdef Py_DEBUG + // gh-147988: Detect uninitialized digits: long_alloc() fills digits with + // 0xFF byte pattern. It's posssible because PyLong_BASE is smaller than + // the maximum value of the C digit type (uint32_t or unsigned short): + // most significan bits are unused by the API. + Py_ssize_t ndigits = _PyLong_DigitCount(obj); + if (ndigits == 0) { + // Check ob_digit[0] digit for the number zero + ndigits = 1; + } + for (Py_ssize_t i = 0; i < ndigits; i++) { + digit d = obj->long_value.ob_digit[i]; + if (d & ~(digit)PyLong_MASK) { + Py_DECREF(obj); + PyErr_Format(PyExc_SystemError, + "PyLongWriter_Finish: digit %zd is uninitialized", + i); + return NULL; + } + } +#endif + // Normalize and get singleton if possible obj = maybe_small_long(long_normalize(obj)); diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 0ad4f02d80bf50..4cbbb7eb7cd0fd 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1216,6 +1216,8 @@ get_native_fmtchar(char *result, const char *fmt) case 'f': size = sizeof(float); break; case 'd': size = sizeof(double); break; case 'e': size = sizeof(float) / 2; break; + case 'F': size = 2*sizeof(float); break; + case 'D': size = 2*sizeof(double); break; case '?': size = sizeof(_Bool); break; case 'P': size = sizeof(void *); break; } @@ -1260,6 +1262,8 @@ get_native_fmtstr(const char *fmt) case 'f': RETURN("f"); case 'd': RETURN("d"); case 'e': RETURN("e"); + case 'F': RETURN("F"); + case 'D': RETURN("D"); case '?': RETURN("?"); case 'P': RETURN("P"); } @@ -1785,7 +1789,7 @@ unpack_single(PyMemoryViewObject *self, const char *ptr, const char *fmt) long long lld; long ld; Py_ssize_t zd; - double d; + double d[2]; unsigned char uc; void *p; @@ -1823,9 +1827,20 @@ unpack_single(PyMemoryViewObject *self, const char *ptr, const char *fmt) case 'N': UNPACK_SINGLE(zu, ptr, size_t); goto convert_zu; /* floats */ - case 'f': UNPACK_SINGLE(d, ptr, float); goto convert_double; - case 'd': UNPACK_SINGLE(d, ptr, double); goto convert_double; - case 'e': d = PyFloat_Unpack2(ptr, endian); goto convert_double; + case 'f': UNPACK_SINGLE(d[0], ptr, float); goto convert_double; + case 'd': UNPACK_SINGLE(d[0], ptr, double); goto convert_double; + case 'e': d[0] = PyFloat_Unpack2(ptr, endian); goto convert_double; + + /* complexes */ + case 'F': + d[0] = PyFloat_Unpack4(ptr, endian); + d[1] = PyFloat_Unpack4(ptr + sizeof(float), endian); + goto convert_double_complex; + + case 'D': + d[0] = PyFloat_Unpack8(ptr, endian); + d[1] = PyFloat_Unpack8(ptr + sizeof(double), endian); + goto convert_double_complex; /* bytes object */ case 'c': goto convert_bytes; @@ -1853,7 +1868,9 @@ unpack_single(PyMemoryViewObject *self, const char *ptr, const char *fmt) convert_zu: return PyLong_FromSize_t(zu); convert_double: - return PyFloat_FromDouble(d); + return PyFloat_FromDouble(d[0]); +convert_double_complex: + return PyComplex_FromDoubles(d[0], d[1]); convert_bool: return PyBool_FromLong(ld); convert_bytes: @@ -1885,6 +1902,7 @@ pack_single(PyMemoryViewObject *self, char *ptr, PyObject *item, const char *fmt long ld; Py_ssize_t zd; double d; + Py_complex c; void *p; #if PY_LITTLE_ENDIAN @@ -1986,6 +2004,25 @@ pack_single(PyMemoryViewObject *self, char *ptr, PyObject *item, const char *fmt } break; + /* complexes */ + case 'F': case 'D': + c = PyComplex_AsCComplex(item); + if (c.real == -1.0 && PyErr_Occurred()) { + goto err_occurred; + } + CHECK_RELEASED_INT_AGAIN(self); + if (fmt[0] == 'D') { + double x[2] = {c.real, c.imag}; + + memcpy(ptr, &x, sizeof(x)); + } + else { + float x[2] = {(float)c.real, (float)c.imag}; + + memcpy(ptr, &x, sizeof(x)); + } + break; + /* bool */ case '?': ld = PyObject_IsTrue(item); @@ -2321,7 +2358,7 @@ memoryview.hex sep: object = NULL An optional single character or byte to separate hex bytes. - bytes_per_sep: int = 1 + bytes_per_sep: Py_ssize_t = 1 How many bytes between separators. Positive values count from the right, negative values count from the left. @@ -2341,8 +2378,8 @@ Return the data in the buffer as a str of hexadecimal numbers. static PyObject * memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, - int bytes_per_sep) -/*[clinic end generated code: output=430ca760f94f3ca7 input=539f6a3a5fb56946]*/ + Py_ssize_t bytes_per_sep) +/*[clinic end generated code: output=c9bb00c7a8e86056 input=dc48a56ed3b058ae]*/ { Py_buffer *src = VIEW_ADDR(self); @@ -2435,7 +2472,7 @@ ptr_from_tuple(const Py_buffer *view, PyObject *tup) if (nindices > view->ndim) { PyErr_Format(PyExc_TypeError, - "cannot index %zd-dimension view with %zd-element tuple", + "cannot index %d-dimension view with %zd-element tuple", view->ndim, nindices); return NULL; } @@ -3023,6 +3060,24 @@ unpack_cmp(const char *p, const char *q, char fmt, return (u == v); } + /* complexes */ + case 'F': + { + float x[2], y[2]; + + memcpy(&x, p, sizeof(x)); + memcpy(&y, q, sizeof(y)); + return (x[0] == y[0]) && (x[1] == y[1]); + } + case 'D': + { + double x[2], y[2]; + + memcpy(&x, p, sizeof(x)); + memcpy(&y, q, sizeof(y)); + return (x[0] == y[0]) && (x[1] == y[1]); + } + /* bytes object */ case 'c': return *p == *q; diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index e3868097c0ba9f..8339e6b91a5e16 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -446,6 +446,7 @@ module_from_def_and_spec( bool seen_m_traverse_slot = false; bool seen_m_clear_slot = false; bool seen_m_free_slot = false; + bool seen_m_abi_slot = false; for (cur_slot = def_like->m_slots; cur_slot && cur_slot->slot; cur_slot++) { // Macro to copy a non-NULL, non-repeatable slot. @@ -555,6 +556,7 @@ module_from_def_and_spec( if (PyABIInfo_Check((PyABIInfo *)cur_slot->value, name) < 0) { goto error; } + seen_m_abi_slot = true; break; DEF_SLOT_CASE(Py_mod_name, char*, m_name) DEF_SLOT_CASE(Py_mod_doc, char*, m_doc) @@ -587,6 +589,14 @@ module_from_def_and_spec( #undef COPY_NONDEF_SLOT #undef COPY_NONNULL_SLOT } + if (!original_def && !seen_m_abi_slot) { + PyErr_Format( + PyExc_SystemError, + "module %s does not define Py_mod_abi," + " which is mandatory for modules defined from slots only.", + name); + goto error; + } #ifdef Py_GIL_DISABLED // For modules created directly from slots (not from a def), we enable diff --git a/Objects/object.c b/Objects/object.c index ae6ad558ff6c37..4db22f372ec3f7 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2032,7 +2032,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, } Py_INCREF(name); - Py_INCREF(tp); + _Py_INCREF_TYPE(tp); PyThreadState *tstate = _PyThreadState_GET(); _PyCStackRef cref; @@ -2107,7 +2107,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, } done: _PyThreadState_PopCStackRef(tstate, &cref); - Py_DECREF(tp); + _Py_DECREF_TYPE(tp); Py_DECREF(name); return res; } @@ -2761,13 +2761,6 @@ _Py_NewReferenceNoTotal(PyObject *op) void _Py_SetImmortalUntracked(PyObject *op) { -#ifdef Py_DEBUG - // For strings, use _PyUnicode_InternImmortal instead. - if (PyUnicode_CheckExact(op)) { - assert(PyUnicode_CHECK_INTERNED(op) == SSTATE_INTERNED_IMMORTAL - || PyUnicode_CHECK_INTERNED(op) == SSTATE_INTERNED_IMMORTAL_STATIC); - } -#endif // Check if already immortal to avoid degrading from static immortal to plain immortal if (_Py_IsImmortal(op)) { return; diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 983bdddbf026a8..e2d5b012955c3e 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -14,6 +14,7 @@ #include // malloc() #include #include // fopen(), fgets(), sscanf() +#include // errno #ifdef WITH_MIMALLOC // Forward declarations of functions used in our mimalloc modifications static void _PyMem_mi_page_clear_qsbr(mi_page_t *page); @@ -572,6 +573,49 @@ _pymalloc_system_hugepage_size(void) } #endif +#if (defined(MS_WINDOWS) && defined(PYMALLOC_USE_HUGEPAGES)) || \ + (defined(PYMALLOC_USE_HUGEPAGES) && defined(ARENAS_USE_MMAP) && defined(MAP_HUGETLB)) +static size_t +_pymalloc_round_up_to_multiple(size_t size, size_t multiple) +{ + if (multiple == 0 || size == 0) { + return size; + } + + size_t remainder = size % multiple; + if (remainder == 0) { + return size; + } + + size_t padding = multiple - remainder; + if (size > SIZE_MAX - padding) { + return 0; + } + return size + padding; +} +#endif + +static size_t +_pymalloc_virtual_alloc_size(size_t size) +{ +#if defined(MS_WINDOWS) && defined(PYMALLOC_USE_HUGEPAGES) + if (_PyRuntime.allocators.use_hugepages) { + SIZE_T large_page_size = GetLargePageMinimum(); + if (large_page_size > 0) { + return _pymalloc_round_up_to_multiple(size, (size_t)large_page_size); + } + } +#elif defined(PYMALLOC_USE_HUGEPAGES) && defined(ARENAS_USE_MMAP) && defined(MAP_HUGETLB) + if (_PyRuntime.allocators.use_hugepages) { + size_t hp_size = _pymalloc_system_hugepage_size(); + if (hp_size > 0) { + return _pymalloc_round_up_to_multiple(size, hp_size); + } + } +#endif + return size; +} + void * _PyMem_ArenaAlloc(void *Py_UNUSED(ctx), size_t size) { @@ -648,7 +692,11 @@ _PyMem_ArenaFree(void *Py_UNUSED(ctx), void *ptr, if (ptr == NULL) { return; } - munmap(ptr, size); + if (munmap(ptr, size) < 0) { + _Py_FatalErrorFormat(__func__, + "munmap(%p, %zu) failed with errno %d", + ptr, size, errno); + } #else free(ptr); #endif @@ -1128,13 +1176,19 @@ PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) void * _PyObject_VirtualAlloc(size_t size) { - return _PyObject_Arena.alloc(_PyObject_Arena.ctx, size); + size_t alloc_size = _pymalloc_virtual_alloc_size(size); + if (alloc_size == 0 && size != 0) { + return NULL; + } + return _PyObject_Arena.alloc(_PyObject_Arena.ctx, alloc_size); } void _PyObject_VirtualFree(void *obj, size_t size) { - _PyObject_Arena.free(_PyObject_Arena.ctx, obj, size); + size_t alloc_size = _pymalloc_virtual_alloc_size(size); + assert(alloc_size != 0 || size == 0); + _PyObject_Arena.free(_PyObject_Arena.ctx, obj, alloc_size); } diff --git a/Objects/odictobject.c b/Objects/odictobject.c index 25928028919c9c..b391283e83795d 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -906,7 +906,7 @@ odict_or(PyObject *left, PyObject *right) type = Py_TYPE(right); other = left; } - if (!PyDict_Check(other)) { + if (!PyAnyDict_Check(other)) { Py_RETURN_NOTIMPLEMENTED; } PyObject *new = PyObject_CallOneArg((PyObject*)type, left); @@ -1156,7 +1156,7 @@ static PyObject * OrderedDict_popitem_impl(PyODictObject *self, int last) /*[clinic end generated code: output=98e7d986690d49eb input=8aafc7433e0a40e7]*/ { - PyObject *key, *value, *item = NULL; + PyObject *key, *value; _ODictNode *node; /* pull the item */ @@ -1169,12 +1169,11 @@ OrderedDict_popitem_impl(PyODictObject *self, int last) node = last ? _odict_LAST(self) : _odict_FIRST(self); key = Py_NewRef(_odictnode_KEY(node)); value = _odict_popkey_hash((PyObject *)self, key, NULL, _odictnode_HASH(node)); - if (value == NULL) + if (value == NULL) { + Py_DECREF(key); return NULL; - item = PyTuple_Pack(2, key, value); - Py_DECREF(key); - Py_DECREF(value); - return item; + } + return _PyTuple_FromPairSteal(key, value); } /* keys() */ @@ -1807,7 +1806,7 @@ odictiter_iternext_lock_held(PyObject *op) if (!PyErr_Occurred()) PyErr_SetObject(PyExc_KeyError, key); Py_DECREF(key); - goto done; + goto error; } /* Handle the values case. */ @@ -1828,21 +1827,19 @@ odictiter_iternext_lock_held(PyObject *op) // bpo-42536: The GC may have untracked this result tuple. Since we're // recycling it, make sure it's tracked again: _PyTuple_Recycle(result); + PyTuple_SET_ITEM(result, 0, key); /* steals reference */ + PyTuple_SET_ITEM(result, 1, value); /* steals reference */ } else { - result = PyTuple_New(2); + result = _PyTuple_FromPairSteal(key, value); if (result == NULL) { - Py_DECREF(key); - Py_DECREF(value); - goto done; + goto error; } } - PyTuple_SET_ITEM(result, 0, key); /* steals reference */ - PyTuple_SET_ITEM(result, 1, value); /* steals reference */ return result; -done: +error: Py_CLEAR(di->di_current); Py_CLEAR(di->di_odict); return NULL; @@ -1933,7 +1930,7 @@ odictiter_new(PyODictObject *od, int kind) return NULL; if ((kind & _odict_ITER_ITEMS) == _odict_ITER_ITEMS) { - di->di_result = PyTuple_Pack(2, Py_None, Py_None); + di->di_result = _PyTuple_FromPairSteal(Py_None, Py_None); if (di->di_result == NULL) { Py_DECREF(di); return NULL; @@ -2271,7 +2268,7 @@ static int mutablemapping_update_arg(PyObject *self, PyObject *arg) { int res = 0; - if (PyDict_CheckExact(arg)) { + if (PyAnyDict_CheckExact(arg)) { PyObject *items = PyDict_Items(arg); if (items == NULL) { return -1; diff --git a/Objects/setobject.c b/Objects/setobject.c index ae6c1d1248d2fc..1e630563604552 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -3023,14 +3023,14 @@ PySet_Contains(PyObject *anyset, PyObject *key) PyErr_BadInternalCall(); return -1; } - if (PyFrozenSet_CheckExact(anyset)) { - return set_contains_key((PySetObject *)anyset, key); + + PySetObject *so = (PySetObject *)anyset; + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + set_unhashable_type(key); + return -1; } - int rv; - Py_BEGIN_CRITICAL_SECTION(anyset); - rv = set_contains_key((PySetObject *)anyset, key); - Py_END_CRITICAL_SECTION(); - return rv; + return set_contains_entry(so, key, hash); } int diff --git a/Objects/stringlib/unicode_format.h b/Objects/stringlib/unicode_format.h index ff32db65b11a0b..c9c46283840d18 100644 --- a/Objects/stringlib/unicode_format.h +++ b/Objects/stringlib/unicode_format.h @@ -4,6 +4,7 @@ #include "pycore_complexobject.h" // _PyComplex_FormatAdvancedWriter() #include "pycore_floatobject.h" // _PyFloat_FormatAdvancedWriter() +#include "pycore_tuple.h" // _PyTuple_FromPairSteal /************************************************************************/ /*********** Global data structures and forward declarations *********/ @@ -1172,7 +1173,7 @@ fieldnameiter_next(PyObject *op) is_attr_obj = PyBool_FromLong(is_attr); if (is_attr_obj == NULL) - goto done; + goto error; /* either an integer or a string */ if (idx != -1) @@ -1180,12 +1181,12 @@ fieldnameiter_next(PyObject *op) else obj = SubString_new_object(&name); if (obj == NULL) - goto done; + goto error; /* return a tuple of values */ - result = PyTuple_Pack(2, is_attr_obj, obj); + return _PyTuple_FromPairSteal(is_attr_obj, obj); - done: + error: Py_XDECREF(is_attr_obj); Py_XDECREF(obj); return result; @@ -1262,7 +1263,7 @@ formatter_field_name_split(PyObject *Py_UNUSED(module), PyObject *self) first_obj in that case. */ if (!field_name_split((PyObject*)self, 0, PyUnicode_GET_LENGTH(self), &first, &first_idx, &it->it_field, NULL)) - goto done; + goto error; /* first becomes an integer, if possible; else a string */ if (first_idx != -1) @@ -1271,12 +1272,12 @@ formatter_field_name_split(PyObject *Py_UNUSED(module), PyObject *self) /* convert "first" into a string object */ first_obj = SubString_new_object(&first); if (first_obj == NULL) - goto done; + goto error; /* return a tuple of values */ - result = PyTuple_Pack(2, first_obj, it); + return _PyTuple_FromPairSteal(first_obj, (PyObject *)it); -done: +error: Py_XDECREF(it); Py_XDECREF(first_obj); return result; diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 01afa53e15cd5d..ee6320e6ca3cfe 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -234,6 +234,23 @@ _PyTuple_FromPairSteal(PyObject *first, PyObject *second) /* Methods */ +/* + Free of a tuple where all contents have been stolen and + is now untracked by GC. This operation is thus non-escaping. + */ +void +_PyStolenTuple_Free(PyObject *obj) +{ + assert(PyTuple_CheckExact(obj)); + PyTupleObject *op = _PyTuple_CAST(obj); + assert(Py_SIZE(op) != 0); + assert(!_PyObject_GC_IS_TRACKED(obj)); + // This will abort on the empty singleton (if there is one). + if (!maybe_freelist_push(op)) { + PyTuple_Type.tp_free((PyObject *)op); + } +} + static void tuple_dealloc(PyObject *self) { diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 7b4318e79fb2be..0ac5377d168812 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -19,6 +19,7 @@ #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_symtable.h" // _Py_Mangle() +#include "pycore_tuple.h" // _PyTuple_FromPair #include "pycore_typeobject.h" // struct type_cache #include "pycore_unicodeobject.h" // _PyUnicode_Copy #include "pycore_unionobject.h" // _Py_union_type_or @@ -1349,6 +1350,35 @@ _PyType_LookupByVersion(unsigned int version) #ifdef Py_GIL_DISABLED return NULL; #else + switch (version) { + case _Py_TYPE_VERSION_INT: + return &PyLong_Type; + case _Py_TYPE_VERSION_FLOAT: + return &PyFloat_Type; + case _Py_TYPE_VERSION_LIST: + return &PyList_Type; + case _Py_TYPE_VERSION_TUPLE: + return &PyTuple_Type; + case _Py_TYPE_VERSION_STR: + return &PyUnicode_Type; + case _Py_TYPE_VERSION_SET: + return &PySet_Type; + case _Py_TYPE_VERSION_FROZEN_SET: + return &PyFrozenSet_Type; + case _Py_TYPE_VERSION_DICT: + return &PyDict_Type; + case _Py_TYPE_VERSION_BYTEARRAY: + return &PyByteArray_Type; + case _Py_TYPE_VERSION_BYTES: + return &PyBytes_Type; + case _Py_TYPE_VERSION_COMPLEX: + return &PyComplex_Type; + case _Py_TYPE_VERSION_FROZENDICT: + return &PyFrozenDict_Type; + default: + break; + } + PyInterpreterState *interp = _PyInterpreterState_GET(); PyTypeObject **slot = interp->types.type_version_cache @@ -1782,7 +1812,7 @@ mro_hierarchy_for_complete_type(PyTypeObject *type, PyObject *temp) tuple = PyTuple_Pack(3, type, new_mro, old_mro); } else { - tuple = PyTuple_Pack(2, type, new_mro); + tuple = _PyTuple_FromPair((PyObject *)type, new_mro); } if (tuple != NULL) { @@ -5154,28 +5184,28 @@ check_basicsize_includes_size_and_offsets(PyTypeObject* type) if (type->tp_base && type->tp_base->tp_basicsize > type->tp_basicsize) { PyErr_Format(PyExc_TypeError, - "tp_basicsize for type '%s' (%d) is too small for base '%s' (%d)", + "tp_basicsize for type '%s' (%zd) is too small for base '%s' (%zd)", type->tp_name, type->tp_basicsize, type->tp_base->tp_name, type->tp_base->tp_basicsize); return 0; } if (type->tp_weaklistoffset + (Py_ssize_t)sizeof(PyObject*) > max) { PyErr_Format(PyExc_TypeError, - "weaklist offset %d is out of bounds for type '%s' (tp_basicsize = %d)", + "weaklist offset %zd is out of bounds for type '%s' (tp_basicsize = %zd)", type->tp_weaklistoffset, type->tp_name, type->tp_basicsize); return 0; } if (type->tp_dictoffset + (Py_ssize_t)sizeof(PyObject*) > max) { PyErr_Format(PyExc_TypeError, - "dict offset %d is out of bounds for type '%s' (tp_basicsize = %d)", + "dict offset %zd is out of bounds for type '%s' (tp_basicsize = %zd)", type->tp_dictoffset, type->tp_name, type->tp_basicsize); return 0; } if (type->tp_vectorcall_offset + (Py_ssize_t)sizeof(vectorcallfunc*) > max) { PyErr_Format(PyExc_TypeError, - "vectorcall offset %d is out of bounds for type '%s' (tp_basicsize = %d)", + "vectorcall offset %zd is out of bounds for type '%s' (tp_basicsize = %zd)", type->tp_vectorcall_offset, type->tp_name, type->tp_basicsize); return 0; @@ -7826,7 +7856,7 @@ object_getstate_default(PyObject *obj, int required) if (PyDict_GET_SIZE(slots) > 0) { PyObject *state2; - state2 = PyTuple_Pack(2, state, slots); + state2 = _PyTuple_FromPair(state, slots); Py_DECREF(state); if (state2 == NULL) { Py_DECREF(slotnames); @@ -9315,6 +9345,7 @@ type_ready_post_checks(PyTypeObject *type) PyErr_Format(PyExc_SystemError, "type %s has a tp_dictoffset that is too small", type->tp_name); + return -1; } } return 0; @@ -11571,7 +11602,7 @@ static pytype_slotdef slotdefs[] = { /* Stores the number of times where slotdefs has elements with same name. This counter precalculated by _PyType_InitSlotDefs() when the main interpreter starts. */ -static uint8_t slotdefs_name_counts[Py_ARRAY_LENGTH(slotdefs)]; +static uint8_t slotdefs_dups[Py_ARRAY_LENGTH(slotdefs)][1 + MAX_EQUIV]; /* Given a type pointer and an offset gotten from a slotdef entry, return a pointer to the actual slot. This is not quite the same as simply adding @@ -11738,11 +11769,22 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p, pytype_slotdef **next_p, ((PyWrapperDescrObject *)descr)->d_base->name_strobj == p->name_strobj) { void **tptr; size_t index = (p - slotdefs); - if (slotdefs_name_counts[index] == 1) { - tptr = slotptr(type, p->offset); + if (slotdefs_dups[index][0] > 1) { + tptr = NULL; + for (size_t i = 1; i <= slotdefs_dups[index][0]; i++) { + pytype_slotdef *q = &slotdefs[slotdefs_dups[index][i]]; + void **qptr = slotptr(type, q->offset); + if (qptr == NULL || *qptr == NULL) + continue; + if (tptr != NULL) { + tptr = NULL; + break; + } + tptr = qptr; + } } else { - tptr = NULL; + tptr = slotptr(type, offset); } if (tptr == NULL || tptr == ptr) @@ -12004,7 +12046,7 @@ _PyType_InitSlotDefs(PyInterpreterState *interp) Py_CLEAR(bytearray); } - memset(slotdefs_name_counts, 0, sizeof(slotdefs_name_counts)); + memset(slotdefs_dups, -1, sizeof(slotdefs_dups)); Py_ssize_t pos = 0; PyObject *key = NULL; @@ -12014,7 +12056,7 @@ _PyType_InitSlotDefs(PyInterpreterState *interp) uint8_t n = data[0]; for (uint8_t i = 0; i < n; i++) { uint8_t idx = data[i + 1]; - slotdefs_name_counts[idx] = n; + memcpy(&slotdefs_dups[idx], data, sizeof(uint8_t) * (n + 1)); } } diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 0a260f4c10278c..b5413ee37a9358 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -2,6 +2,7 @@ #include "Python.h" #include "pycore_interpframe.h" // _PyInterpreterFrame #include "pycore_object.h" // _PyObject_GC_TRACK/UNTRACK, PyAnnotateFormat +#include "pycore_tuple.h" // _PyTuple_FromPair #include "pycore_typevarobject.h" #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() #include "pycore_unionobject.h" // _Py_union_type_or, _Py_union_from_tuple @@ -373,7 +374,7 @@ type_check(PyObject *arg, const char *msg) static PyObject * make_union(PyObject *self, PyObject *other) { - PyObject *args = PyTuple_Pack(2, self, other); + PyObject *args = _PyTuple_FromPair(self, other); if (args == NULL) { return NULL; } @@ -817,7 +818,7 @@ typevar_typing_prepare_subst_impl(typevarobject *self, PyObject *alias, } Py_DECREF(params); PyErr_Format(PyExc_TypeError, - "Too few arguments for %S; actual %d, expected at least %d", + "Too few arguments for %S; actual %zd, expected at least %zd", alias, args_len, i + 1); return NULL; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index d51a95c69a93b3..a0a26a75129929 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5580,15 +5580,14 @@ _Py_EncodeUTF8Ex(const wchar_t *text, char **str, size_t *error_pos, Py_ssize_t ch_pos = i; Py_UCS4 ch = text[i]; i++; -#if Py_UNICODE_SIZE == 2 - if (Py_UNICODE_IS_HIGH_SURROGATE(ch) + if (sizeof(wchar_t) == 2 + && Py_UNICODE_IS_HIGH_SURROGATE(ch) && i < len && Py_UNICODE_IS_LOW_SURROGATE(text[i])) { ch = Py_UNICODE_JOIN_SURROGATES(ch, text[i]); i++; } -#endif if (ch < 0x80) { /* Encode ASCII */ @@ -8351,7 +8350,7 @@ charmap_decode_mapping(const char *s, goto Undefined; if (value < 0 || value > MAX_UNICODE) { PyErr_Format(PyExc_TypeError, - "character mapping must be in range(0x%x)", + "character mapping must be in range(0x%lx)", (unsigned long)MAX_UNICODE + 1); goto onError; } @@ -9142,8 +9141,8 @@ charmaptranslate_lookup(Py_UCS4 c, PyObject *mapping, PyObject **result, Py_UCS4 long value = PyLong_AsLong(x); if (value < 0 || value > MAX_UNICODE) { PyErr_Format(PyExc_ValueError, - "character mapping must be in range(0x%x)", - MAX_UNICODE+1); + "character mapping must be in range(0x%lx)", + (unsigned long)MAX_UNICODE + 1); Py_DECREF(x); return -1; } @@ -12562,7 +12561,6 @@ PyUnicode_Replace(PyObject *str, } /*[clinic input] -@permit_long_docstring_body str.replace as unicode_replace old: unicode @@ -12574,14 +12572,14 @@ str.replace as unicode_replace Return a copy with all occurrences of substring old replaced by new. -If the optional argument count is given, only the first count occurrences are -replaced. +If count is given, only the first count occurrences are replaced. +If count is not specified or -1, then all occurrences are replaced. [clinic start generated code]*/ static PyObject * unicode_replace_impl(PyObject *self, PyObject *old, PyObject *new, Py_ssize_t count) -/*[clinic end generated code: output=b63f1a8b5eebf448 input=f27ca92ac46b65a1]*/ +/*[clinic end generated code: output=b63f1a8b5eebf448 input=d15a6886b05e2edc]*/ { return replace(self, old, new, count); } @@ -14188,8 +14186,11 @@ immortalize_interned(PyObject *s) _Py_DecRefTotal(_PyThreadState_GET()); } #endif - FT_ATOMIC_STORE_UINT8_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_IMMORTAL); _Py_SetImmortal(s); + // The switch to SSTATE_INTERNED_IMMORTAL must be the last thing done here + // to synchronize with the check in intern_common() that avoids locking if + // the string is already immortal. + FT_ATOMIC_STORE_UINT8(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_IMMORTAL); } static /* non-null */ PyObject* @@ -14271,6 +14272,23 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, assert(interned != NULL); #ifdef Py_GIL_DISABLED # define INTERN_MUTEX &_Py_INTERP_CACHED_OBJECT(interp, interned_mutex) + // Lock-free fast path: check if there's already an interned copy that + // is in its final immortal state. + PyObject *r; + int res = PyDict_GetItemRef(interned, s, &r); + if (res < 0) { + PyErr_Clear(); + return s; + } + if (res > 0) { + unsigned int state = _Py_atomic_load_uint8(&_PyUnicode_STATE(r).interned); + if (state == SSTATE_INTERNED_IMMORTAL) { + Py_DECREF(s); + return r; + } + // Not yet fully interned; fall through to the locking path. + Py_DECREF(r); + } #endif FT_MUTEX_LOCK(INTERN_MUTEX); PyObject *t; @@ -14308,7 +14326,7 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, Py_DECREF(s); Py_DECREF(s); } - FT_ATOMIC_STORE_UINT8_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_MORTAL); + FT_ATOMIC_STORE_UINT8(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_MORTAL); /* INTERNED_MORTAL -> INTERNED_IMMORTAL (if needed) */ @@ -14929,6 +14947,7 @@ static PyMethodDef _string_methods[] = { }; static PyModuleDef_Slot module_slots[] = { + _Py_ABI_SLOT, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} diff --git a/Objects/unionobject.c b/Objects/unionobject.c index a47d6193d70889..d33d581f049c5b 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -61,7 +61,7 @@ union_hash(PyObject *self) } // The unhashable values somehow became hashable again. Still raise // an error. - PyErr_Format(PyExc_TypeError, "union contains %d unhashable elements", n); + PyErr_Format(PyExc_TypeError, "union contains %zd unhashable elements", n); return -1; } return PyObject_Hash(alias->hashable_args); diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 5ad20d49fa4b31..71a164fbec5a06 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1989,6 +1989,7 @@ def visitModule(self, mod): self.emit("", 0) self.emit(""" static PyModuleDef_Slot astmodule_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, astmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Apple/.ruff.toml b/Platforms/Apple/.ruff.toml similarity index 88% rename from Apple/.ruff.toml rename to Platforms/Apple/.ruff.toml index 4cdc39ebee4be9..f5d74fdb6afe87 100644 --- a/Apple/.ruff.toml +++ b/Platforms/Apple/.ruff.toml @@ -1,4 +1,4 @@ -extend = "../.ruff.toml" # Inherit the project-wide settings +extend = "../../.ruff.toml" # Inherit the project-wide settings [format] preview = true diff --git a/Apple/__main__.py b/Platforms/Apple/__main__.py similarity index 94% rename from Apple/__main__.py rename to Platforms/Apple/__main__.py index 3261f368a88fc0..44a991c6c20a93 100644 --- a/Apple/__main__.py +++ b/Platforms/Apple/__main__.py @@ -10,7 +10,7 @@ # # The simplest entry point is: # -# $ python Apple ci iOS +# $ python Platforms/Apple ci iOS # # which will: # * Clean any pre-existing build artefacts @@ -57,7 +57,7 @@ ArgsT = Sequence[str | Path] SCRIPT_NAME = Path(__file__).name -PYTHON_DIR = Path(__file__).resolve().parent.parent +PYTHON_DIR = Path(__file__).resolve().parent.parent.parent CROSS_BUILD_DIR = PYTHON_DIR / "cross-build" @@ -140,7 +140,7 @@ def apple_env(host: str) -> EnvironmentT: """Construct an Apple development environment for the given host.""" env = { "PATH": ":".join([ - str(PYTHON_DIR / "Apple/iOS/Resources/bin"), + str(PYTHON_DIR / "Platforms/Apple/iOS/Resources/bin"), str(subdir(host) / "prefix"), "/usr/bin", "/bin", @@ -173,8 +173,11 @@ def all_host_triples(platform: str) -> list[str]: return triples -def clean(context: argparse.Namespace, target: str = "all") -> None: +def clean(context: argparse.Namespace, target: str | None = None) -> None: """The implementation of the "clean" command.""" + if target is None: + target = context.host + # If we're explicitly targeting the build, there's no platform or # distribution artefacts. If we're cleaning tests, we keep all built # artefacts. Otherwise, the built artefacts must be dirty, so we remove @@ -377,7 +380,12 @@ def configure_host_python( with group(f"Downloading dependencies ({host})"): if not prefix_dir.exists(): prefix_dir.mkdir() - unpack_deps(context.platform, host, prefix_dir, context.cache_dir) + cache_dir = ( + Path(context.cache_dir).resolve() + if context.cache_dir + else CROSS_BUILD_DIR / "downloads" + ) + unpack_deps(context.platform, host, prefix_dir, cache_dir) else: print("Dependencies already installed") @@ -432,7 +440,10 @@ def framework_path(host_triple: str, multiarch: str) -> Path: :param host_triple: The host triple (e.g., arm64-apple-ios-simulator) :param multiarch: The multiarch identifier (e.g., arm64-simulator) """ - return CROSS_BUILD_DIR / f"{host_triple}/Apple/iOS/Frameworks/{multiarch}" + return ( + CROSS_BUILD_DIR + / f"{host_triple}/Platforms/Apple/iOS/Frameworks/{multiarch}" + ) def package_version(prefix_path: Path) -> str: @@ -616,7 +627,7 @@ def create_xcframework(platform: str) -> str: # Copy in the cross-architecture pyconfig.h shutil.copy( - PYTHON_DIR / f"Apple/{platform}/Resources/pyconfig.h", + PYTHON_DIR / f"Platforms/Apple/{platform}/Resources/pyconfig.h", slice_framework / "Headers/pyconfig.h", ) @@ -653,7 +664,7 @@ def create_xcframework(platform: str) -> str: host_path = ( CROSS_BUILD_DIR / host_triple - / "Apple/iOS/Frameworks" + / "Platforms/Apple/iOS/Frameworks" / multiarch ) host_framework = host_path / "Python.framework" @@ -683,7 +694,7 @@ def create_xcframework(platform: str) -> str: print(" - build tools") shutil.copytree( - PYTHON_DIR / "Apple/testbed/Python.xcframework/build", + PYTHON_DIR / "Platforms/Apple/testbed/Python.xcframework/build", package_path / "Python.xcframework/build", ) @@ -703,7 +714,7 @@ def package(context: argparse.Namespace) -> None: print() run([ sys.executable, - "Apple/testbed", + "Platforms/Apple/testbed", "clone", "--platform", context.platform, @@ -760,7 +771,7 @@ def build(context: argparse.Namespace, host: str | None = None) -> None: ]: step(context, host=step_host) - if host in {"all", "hosts"}: + if host == "all": package(context) @@ -798,13 +809,13 @@ def test(context: argparse.Namespace, host: str | None = None) -> None: # noqa: framework_path = ( CROSS_BUILD_DIR / host - / f"Apple/{context.platform}" + / f"Platforms/Apple/{context.platform}" / f"Frameworks/{apple_multiarch(host)}" ) run([ sys.executable, - "Apple/testbed", + "Platforms/Apple/testbed", "clone", "--platform", context.platform, @@ -828,9 +839,10 @@ def test(context: argparse.Namespace, host: str | None = None) -> None: # noqa: + [ "--", "test", - f"--{context.ci_mode}-ci", + f"--{context.ci_mode or 'fast'}-ci", "--single-process", "--no-randomize", + "--pythoninfo", # Timeout handling requires subprocesses; explicitly setting # the timeout to -1 disables the faulthandler. "--timeout=-1", @@ -894,7 +906,7 @@ def parse_args() -> argparse.Namespace: configure_build = subcommands.add_parser( "configure-build", help="Run `configure` for the build Python" ) - subcommands.add_parser( + make_build = subcommands.add_parser( "make-build", help="Run `make` for the build Python" ) configure_host = subcommands.add_parser( @@ -950,6 +962,31 @@ def parse_args() -> argparse.Namespace: ), ) + # --cross-build-dir argument + for cmd in [ + clean, + configure_build, + make_build, + configure_host, + make_host, + build, + package, + test, + ci, + ]: + cmd.add_argument( + "--cross-build-dir", + action="store", + default=os.environ.get("CROSS_BUILD_DIR"), + dest="cross_build_dir", + type=Path, + help=( + "Path to the cross-build directory " + f"(default: {CROSS_BUILD_DIR}). Can also be set " + "with the CROSS_BUILD_DIR environment variable." + ), + ) + # --clean option for cmd in [configure_build, configure_host, build, package, test, ci]: cmd.add_argument( @@ -964,7 +1001,7 @@ def parse_args() -> argparse.Namespace: for cmd in [configure_host, build, ci]: cmd.add_argument( "--cache-dir", - default="./cross-build/downloads", + default=os.environ.get("CACHE_DIR"), help="The directory to store cached downloads.", ) @@ -1031,6 +1068,12 @@ def signal_handler(*args): # Process command line arguments context = parse_args() + + # Set the CROSS_BUILD_DIR if an argument was provided + if context.cross_build_dir: + global CROSS_BUILD_DIR + CROSS_BUILD_DIR = context.cross_build_dir.resolve() + dispatch: dict[str, Callable] = { "clean": clean, "configure-build": configure_build_python, diff --git a/Apple/iOS/README.md b/Platforms/Apple/iOS/README.md similarity index 92% rename from Apple/iOS/README.md rename to Platforms/Apple/iOS/README.md index 7ee257b5d648f4..faeeead1df03a2 100644 --- a/Apple/iOS/README.md +++ b/Platforms/Apple/iOS/README.md @@ -52,11 +52,11 @@ portable to machines using other architectures. ### Building a multi-architecture iOS XCframework -The `Apple` subfolder of the Python repository acts as a build script that +The `Platforms/Apple` subfolder of the Python repository acts as a build script that can be used to coordinate the compilation of a complete iOS XCframework. To use it, run:: - python Apple build iOS + python Platforms/Apple build iOS This will: @@ -69,7 +69,7 @@ This will: the `Python.xcframework`, plus a copy of the Testbed app pre-configured to use the XCframework. -The `Apple` build script has other entry points that will perform the +The `Platforms/Apple` build script has other entry points that will perform the individual parts of the overall `build` target, plus targets to test the build, clean the `cross-build` folder of iOS build products, and perform a complete "build and test" CI run. The `--clean` flag can also be used on @@ -78,7 +78,7 @@ building. ### Building a single-architecture framework -If you're using the `Apple` build script, you won't need to build +If you're using the `Platforms/Apple` build script, you won't need to build individual frameworks. However, if you do need to manually configure an iOS Python build for a single framework, the following options are available. @@ -100,7 +100,7 @@ Python build for a single framework, the following options are available. > [!NOTE] > Unless you know what you're doing, changing the name of the Python > framework on iOS is not advised. If you use this option, you won't be able - > to run the `Apple` build script without making significant manual + > to run the `Platforms/Apple` build script without making significant manual > alterations, and you won't be able to use any binary packages unless you > compile them yourself using your own framework name. @@ -119,7 +119,7 @@ provide the `--enable-framework` flag when configuring the build. The build also requires the use of cross-compilation. The minimal commands for building Python for the ARM64 iOS simulator will look something like: ``` -export PATH="$(pwd)/Apple/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" +export PATH="$(pwd)/Platforms/Apple/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" ./configure \ --enable-framework \ --host=arm64-apple-ios-simulator \ @@ -131,7 +131,7 @@ make install In this invocation: -* `Apple/iOS/Resources/bin` has been added to the path, providing some shims for the +* `Platforms/Apple/iOS/Resources/bin` has been added to the path, providing some shims for the compilers and linkers needed by the build. Xcode requires the use of `xcrun` to invoke compiler tooling. However, if `xcrun` is pre-evaluated and the result passed to `configure`, these results can embed user- and @@ -141,7 +141,7 @@ In this invocation: cause significant problems with many C configuration systems which assume that `CC` will be a single executable. - To work around this problem, the `Apple/iOS/Resources/bin` folder contains some + To work around this problem, the `Platforms/Apple/iOS/Resources/bin` folder contains some wrapper scripts that present as simple compilers and linkers, but wrap underlying calls to `xcrun`. This allows configure to use a `CC` definition without spaces, and without user- or version-specific paths, while @@ -222,7 +222,7 @@ simulator build with a deployment target of 15.4. Once you have a built an XCframework, you can test that framework by running: - $ python Apple test iOS + $ python Platforms/Apple test iOS This test will attempt to find an "SE-class" simulator (i.e., an iPhone SE, or iPhone 16e, or similar), and run the test suite on the most recent version of @@ -237,7 +237,7 @@ environment variable will be exposed to the iOS process at runtime. ### Testing a single-architecture framework -The `Apple/testbed` folder that contains an Xcode project that is able to run +The `Platforms/Apple/testbed` folder that contains an Xcode project that is able to run the Python test suite on Apple platforms. This project converts the Python test suite into a single test case in Xcode's XCTest framework. The single XCTest passes if the test suite passes. @@ -245,7 +245,7 @@ passes if the test suite passes. To run the test suite, configure a Python build for an iOS simulator (i.e., `--host=arm64-apple-ios-simulator` or `--host=x86_64-apple-ios-simulator` ), specifying a framework build (i.e. `--enable-framework`). Ensure that your -`PATH` has been configured to include the `Apple/iOS/Resources/bin` folder and +`PATH` has been configured to include the `Platforms/Apple/iOS/Resources/bin` folder and exclude any non-iOS tools, then run: ``` make all @@ -269,9 +269,9 @@ project, and then boot and prepare the iOS simulator. ### Debugging test failures -Running `python Apple test iOS` generates a standalone version of the -`Apple/testbed` project, and runs the full test suite. It does this using -`Apple/testbed` itself - the folder is an executable module that can be used +Running `python Platforms/Apple test iOS` generates a standalone version of the +`Platforms/Apple/testbed` project, and runs the full test suite. It does this using +`Platforms/Apple/testbed` itself - the folder is an executable module that can be used to create and run a clone of the testbed project. The standalone version of the testbed will be created in a directory named `cross-build/iOS-testbed.`. @@ -287,7 +287,7 @@ testbed clone. If you've built your own XCframework, or you only want to test a single architecture, you can construct a standalone testbed instance by running: ``` -python Apple/testbed clone --platform iOS --framework my-testbed +python Platforms/Apple/testbed clone --platform iOS --framework my-testbed ``` The framework path can be the path path to a `Python.xcframework`, or the diff --git a/Apple/iOS/Resources/Info.plist.in b/Platforms/Apple/iOS/Resources/Info.plist.in similarity index 100% rename from Apple/iOS/Resources/Info.plist.in rename to Platforms/Apple/iOS/Resources/Info.plist.in diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-ar b/Platforms/Apple/iOS/Resources/bin/arm64-apple-ios-ar similarity index 100% rename from Apple/iOS/Resources/bin/arm64-apple-ios-ar rename to Platforms/Apple/iOS/Resources/bin/arm64-apple-ios-ar diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-clang b/Platforms/Apple/iOS/Resources/bin/arm64-apple-ios-clang similarity index 100% rename from Apple/iOS/Resources/bin/arm64-apple-ios-clang rename to Platforms/Apple/iOS/Resources/bin/arm64-apple-ios-clang diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-clang++ b/Platforms/Apple/iOS/Resources/bin/arm64-apple-ios-clang++ similarity index 100% rename from Apple/iOS/Resources/bin/arm64-apple-ios-clang++ rename to Platforms/Apple/iOS/Resources/bin/arm64-apple-ios-clang++ diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-cpp b/Platforms/Apple/iOS/Resources/bin/arm64-apple-ios-cpp similarity index 100% rename from Apple/iOS/Resources/bin/arm64-apple-ios-cpp rename to Platforms/Apple/iOS/Resources/bin/arm64-apple-ios-cpp diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-ar b/Platforms/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-ar similarity index 100% rename from Apple/iOS/Resources/bin/arm64-apple-ios-simulator-ar rename to Platforms/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-ar diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang b/Platforms/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang similarity index 100% rename from Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang rename to Platforms/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ b/Platforms/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ similarity index 100% rename from Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ rename to Platforms/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-cpp b/Platforms/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-cpp similarity index 100% rename from Apple/iOS/Resources/bin/arm64-apple-ios-simulator-cpp rename to Platforms/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-cpp diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-strip b/Platforms/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-strip similarity index 100% rename from Apple/iOS/Resources/bin/arm64-apple-ios-simulator-strip rename to Platforms/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-strip diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-strip b/Platforms/Apple/iOS/Resources/bin/arm64-apple-ios-strip similarity index 100% rename from Apple/iOS/Resources/bin/arm64-apple-ios-strip rename to Platforms/Apple/iOS/Resources/bin/arm64-apple-ios-strip diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-ar b/Platforms/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-ar similarity index 100% rename from Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-ar rename to Platforms/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-ar diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang b/Platforms/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang similarity index 100% rename from Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang rename to Platforms/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ b/Platforms/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ similarity index 100% rename from Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ rename to Platforms/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp b/Platforms/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp similarity index 100% rename from Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp rename to Platforms/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-strip b/Platforms/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-strip similarity index 100% rename from Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-strip rename to Platforms/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-strip diff --git a/Apple/iOS/Resources/pyconfig.h b/Platforms/Apple/iOS/Resources/pyconfig.h similarity index 100% rename from Apple/iOS/Resources/pyconfig.h rename to Platforms/Apple/iOS/Resources/pyconfig.h diff --git a/Apple/testbed/Python.xcframework/Info.plist b/Platforms/Apple/testbed/Python.xcframework/Info.plist similarity index 100% rename from Apple/testbed/Python.xcframework/Info.plist rename to Platforms/Apple/testbed/Python.xcframework/Info.plist diff --git a/Apple/testbed/Python.xcframework/build/iOS-dylib-Info-template.plist b/Platforms/Apple/testbed/Python.xcframework/build/iOS-dylib-Info-template.plist similarity index 100% rename from Apple/testbed/Python.xcframework/build/iOS-dylib-Info-template.plist rename to Platforms/Apple/testbed/Python.xcframework/build/iOS-dylib-Info-template.plist diff --git a/Apple/testbed/Python.xcframework/build/utils.sh b/Platforms/Apple/testbed/Python.xcframework/build/utils.sh similarity index 95% rename from Apple/testbed/Python.xcframework/build/utils.sh rename to Platforms/Apple/testbed/Python.xcframework/build/utils.sh index e7155d8b30e213..e54471f68b7cb2 100755 --- a/Apple/testbed/Python.xcframework/build/utils.sh +++ b/Platforms/Apple/testbed/Python.xcframework/build/utils.sh @@ -42,11 +42,11 @@ install_stdlib() { # If the XCframework has a shared lib folder, then it's a full framework. # Copy both the common and slice-specific part of the lib directory. # Otherwise, it's a single-arch framework; use the "full" lib folder. + # Don't include any libpython symlink; that can't be included at runtime if [ -d "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/lib" ]; then - rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" - rsync -au "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib-$ARCHS/" "$CODESIGNING_FOLDER_PATH/python/lib/" + rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" --exclude 'libpython*.dylib' + rsync -au "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib-$ARCHS/" "$CODESIGNING_FOLDER_PATH/python/lib/" --exclude 'libpython*.dylib' else - # A single-arch framework will have a libpython symlink; that can't be included at runtime rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" --exclude 'libpython*.dylib' fi } @@ -140,7 +140,7 @@ install_python() { shift install_stdlib $PYTHON_XCFRAMEWORK_PATH - PYTHON_VER=$(ls -1 "$CODESIGNING_FOLDER_PATH/python/lib") + PYTHON_VER=$(ls -1 "$CODESIGNING_FOLDER_PATH/python/lib" | grep -E "^python3\.\d+$") echo "Install Python $PYTHON_VER standard library extension modules..." process_dylibs $PYTHON_XCFRAMEWORK_PATH python/lib/$PYTHON_VER/lib-dynload diff --git a/Apple/testbed/Python.xcframework/ios-arm64/README b/Platforms/Apple/testbed/Python.xcframework/ios-arm64/README similarity index 100% rename from Apple/testbed/Python.xcframework/ios-arm64/README rename to Platforms/Apple/testbed/Python.xcframework/ios-arm64/README diff --git a/Apple/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README b/Platforms/Apple/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README similarity index 100% rename from Apple/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README rename to Platforms/Apple/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README diff --git a/Apple/testbed/Testbed.lldbinit b/Platforms/Apple/testbed/Testbed.lldbinit similarity index 100% rename from Apple/testbed/Testbed.lldbinit rename to Platforms/Apple/testbed/Testbed.lldbinit diff --git a/Apple/testbed/TestbedTests/TestbedTests.m b/Platforms/Apple/testbed/TestbedTests/TestbedTests.m similarity index 100% rename from Apple/testbed/TestbedTests/TestbedTests.m rename to Platforms/Apple/testbed/TestbedTests/TestbedTests.m diff --git a/Apple/testbed/__main__.py b/Platforms/Apple/testbed/__main__.py similarity index 100% rename from Apple/testbed/__main__.py rename to Platforms/Apple/testbed/__main__.py diff --git a/Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj b/Platforms/Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj similarity index 100% rename from Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj rename to Platforms/Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj diff --git a/Apple/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme b/Platforms/Apple/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme similarity index 100% rename from Apple/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme rename to Platforms/Apple/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme diff --git a/Apple/testbed/iOSTestbed.xctestplan b/Platforms/Apple/testbed/iOSTestbed.xctestplan similarity index 100% rename from Apple/testbed/iOSTestbed.xctestplan rename to Platforms/Apple/testbed/iOSTestbed.xctestplan diff --git a/Apple/testbed/iOSTestbed/AppDelegate.h b/Platforms/Apple/testbed/iOSTestbed/AppDelegate.h similarity index 100% rename from Apple/testbed/iOSTestbed/AppDelegate.h rename to Platforms/Apple/testbed/iOSTestbed/AppDelegate.h diff --git a/Apple/testbed/iOSTestbed/AppDelegate.m b/Platforms/Apple/testbed/iOSTestbed/AppDelegate.m similarity index 100% rename from Apple/testbed/iOSTestbed/AppDelegate.m rename to Platforms/Apple/testbed/iOSTestbed/AppDelegate.m diff --git a/Apple/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json b/Platforms/Apple/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json similarity index 100% rename from Apple/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json rename to Platforms/Apple/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json diff --git a/Apple/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json b/Platforms/Apple/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json similarity index 100% rename from Apple/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json rename to Platforms/Apple/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json diff --git a/Apple/testbed/iOSTestbed/Assets.xcassets/Contents.json b/Platforms/Apple/testbed/iOSTestbed/Assets.xcassets/Contents.json similarity index 100% rename from Apple/testbed/iOSTestbed/Assets.xcassets/Contents.json rename to Platforms/Apple/testbed/iOSTestbed/Assets.xcassets/Contents.json diff --git a/Apple/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard b/Platforms/Apple/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard similarity index 100% rename from Apple/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard rename to Platforms/Apple/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard diff --git a/Apple/testbed/iOSTestbed/app/README b/Platforms/Apple/testbed/iOSTestbed/app/README similarity index 100% rename from Apple/testbed/iOSTestbed/app/README rename to Platforms/Apple/testbed/iOSTestbed/app/README diff --git a/Apple/testbed/iOSTestbed/app_packages/README b/Platforms/Apple/testbed/iOSTestbed/app_packages/README similarity index 100% rename from Apple/testbed/iOSTestbed/app_packages/README rename to Platforms/Apple/testbed/iOSTestbed/app_packages/README diff --git a/Apple/testbed/iOSTestbed/iOSTestbed-Info.plist b/Platforms/Apple/testbed/iOSTestbed/iOSTestbed-Info.plist similarity index 100% rename from Apple/testbed/iOSTestbed/iOSTestbed-Info.plist rename to Platforms/Apple/testbed/iOSTestbed/iOSTestbed-Info.plist diff --git a/Apple/testbed/iOSTestbed/main.m b/Platforms/Apple/testbed/iOSTestbed/main.m similarity index 100% rename from Apple/testbed/iOSTestbed/main.m rename to Platforms/Apple/testbed/iOSTestbed/main.m diff --git a/Platforms/emscripten/__main__.py b/Platforms/emscripten/__main__.py index 6a7963413da31a..c1eac8005474fd 100644 --- a/Platforms/emscripten/__main__.py +++ b/Platforms/emscripten/__main__.py @@ -350,11 +350,18 @@ def write_library_config(prefix, name, config, quiet): def make_emscripten_libffi(context, working_dir): validate_emsdk_version(context.emsdk_cache) prefix = context.build_paths["prefix_dir"] - libffi_config = load_config_toml()["libffi"] + libffi_config = load_config_toml()["dependencies"]["libffi"] + with open(EMSCRIPTEN_DIR / "make_libffi.sh", "rb") as f: + libffi_config["make_libffi_shasum"] = hashlib.file_digest(f, "sha256").hexdigest() if not should_build_library( prefix, "libffi", libffi_config, context.quiet ): return + + if context.check_up_to_date: + print("libffi out of date, expected to be up to date", file=sys.stderr) + sys.exit(1) + url = libffi_config["url"] version = libffi_config["version"] shasum = libffi_config["shasum"] @@ -378,10 +385,14 @@ def make_emscripten_libffi(context, working_dir): def make_mpdec(context, working_dir): validate_emsdk_version(context.emsdk_cache) prefix = context.build_paths["prefix_dir"] - mpdec_config = load_config_toml()["mpdec"] + mpdec_config = load_config_toml()["dependencies"]["mpdec"] if not should_build_library(prefix, "mpdec", mpdec_config, context.quiet): return + if context.check_up_to_date: + print("libmpdec out of date, expected to be up to date", file=sys.stderr) + sys.exit(1) + url = mpdec_config["url"] version = mpdec_config["version"] shasum = mpdec_config["shasum"] @@ -507,6 +518,10 @@ def configure_emscripten_python(context, working_dir): EMSCRIPTEN_DIR / "node_entry.mjs", working_dir / "node_entry.mjs" ) + shutil.copy( + EMSCRIPTEN_DIR / "streams.mjs", working_dir / "streams.mjs" + ) + node_entry = working_dir / "node_entry.mjs" exec_script = working_dir / "python.sh" exec_script.write_text( @@ -580,6 +595,8 @@ def run_emscripten_python(context): if context.test: args = load_config_toml()["test-args"] + args + elif context.pythoninfo: + args = load_config_toml()["pythoninfo-args"] + args os.execv(str(exec_script), [str(exec_script), *args]) @@ -678,6 +695,14 @@ def main(): help="Build all static library dependencies", ) + for cmd in [make_mpdec_cmd, make_libffi_cmd, make_dependencies_cmd]: + cmd.add_argument( + "--check-up-to-date", + action="store_true", + default=False, + help=("If passed, will fail if dependency is out of date"), + ) + make_build = subcommands.add_parser( "make-build-python", help="Run `make` for the build Python" ) @@ -703,9 +728,15 @@ def main(): action="store_true", default=False, help=( - "If passed, will add the default test arguments to the beginning of the command. " + "Add the default test arguments to the beginning of the command. " "Default arguments loaded from Platforms/emscripten/config.toml" - ) + ), + ) + run.add_argument( + "--pythoninfo", + action="store_true", + default=False, + help="Run -m test.pythoninfo", ) run.add_argument( "args", @@ -713,7 +744,7 @@ def main(): help=( "Arguments to pass to the emscripten Python " "(use '--' to separate from run options)", - ) + ), ) add_cross_build_dir_option(run) @@ -793,6 +824,7 @@ def main(): context = parser.parse_args() context.emsdk_cache = getattr(context, "emsdk_cache", None) context.cross_build_dir = getattr(context, "cross_build_dir", None) + context.check_up_to_date = getattr(context, "check_up_to_date", False) if context.emsdk_cache: context.emsdk_cache = Path(context.emsdk_cache).absolute() diff --git a/Platforms/emscripten/browser_test/package-lock.json b/Platforms/emscripten/browser_test/package-lock.json index 044e3c19ce15f7..978aea0147bc28 100644 --- a/Platforms/emscripten/browser_test/package-lock.json +++ b/Platforms/emscripten/browser_test/package-lock.json @@ -10,18 +10,42 @@ "license": "ISC", "dependencies": { "@playwright/test": "^1.54.1", - "@types/node": "^24.1.0", + "@types/node": "^24.12.0", + "get-port-cli": "^3.0.0", "http-server": "^14.1.1", "playwright": "^1.54.1" } }, + "node_modules/@babel/code-frame": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", + "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.28.5", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@playwright/test": { - "version": "1.54.1", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.54.1.tgz", - "integrity": "sha512-FS8hQ12acieG2dYSksmLOF7BNxnVf2afRJdCuM1eMSxj6QTSE6G4InGF7oApGgDb65MX7AwMVlIkpru0yZA4Xw==", + "version": "1.58.2", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.58.2.tgz", + "integrity": "sha512-akea+6bHYBBfA9uQqSYmlJXn61cTa+jbO87xVLCWbTqbWadRVmhxlXATaOjOgcBaWU4ePo0wB41KMFv3o35IXA==", "license": "Apache-2.0", "dependencies": { - "playwright": "1.54.1" + "playwright": "1.58.2" }, "bin": { "playwright": "cli.js" @@ -30,15 +54,27 @@ "node": ">=18" } }, + "node_modules/@types/minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==", + "license": "MIT" + }, "node_modules/@types/node": { - "version": "24.1.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.1.0.tgz", - "integrity": "sha512-ut5FthK5moxFKH2T1CUOC6ctR67rQRvvHdFLCD2Ql6KXmMuCrjsSsRI9UsLCm9M18BMwClv4pn327UvB7eeO1w==", + "version": "24.12.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.0.tgz", + "integrity": "sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ==", "license": "MIT", "dependencies": { - "undici-types": "~7.8.0" + "undici-types": "~7.16.0" } }, + "node_modules/@types/normalize-package-data": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz", + "integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==", + "license": "MIT" + }, "node_modules/ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -54,6 +90,15 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/arrify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/async": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", @@ -101,6 +146,36 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/camelcase-keys": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-7.0.2.tgz", + "integrity": "sha512-Rjs1H+A9R+Ig+4E/9oyB66UC5Mj9Xq3N//vcLf2WzgdTi/3gUu3Z9KoqmlrEG4VuuLK8wJHofxzdQXz/knhiYg==", + "license": "MIT", + "dependencies": { + "camelcase": "^6.3.0", + "map-obj": "^4.1.0", + "quick-lru": "^5.1.1", + "type-fest": "^1.2.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -161,6 +236,52 @@ } } }, + "node_modules/decamelize": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-5.0.1.tgz", + "integrity": "sha512-VfxadyCECXgQlkoEAjeghAr5gY3Hf+IKjKb+X8tGVDtveCjN+USwprd2q3QXBR9T1+x2DG0XZF5/w+7HAtSaXA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/decamelize-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/decamelize-keys/-/decamelize-keys-1.1.1.tgz", + "integrity": "sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==", + "license": "MIT", + "dependencies": { + "decamelize": "^1.1.0", + "map-obj": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/decamelize-keys/node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/decamelize-keys/node_modules/map-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz", + "integrity": "sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -175,6 +296,15 @@ "node": ">= 0.4" } }, + "node_modules/error-ex": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", + "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, "node_modules/es-define-property": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", @@ -211,6 +341,22 @@ "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", "license": "MIT" }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/follow-redirects": { "version": "1.15.9", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", @@ -278,6 +424,37 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-port": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-6.1.2.tgz", + "integrity": "sha512-BrGGraKm2uPqurfGVj/z97/zv8dPleC6x9JBNRTrDNtCkkRF4rPwrQXFgL7+I+q8QSdU4ntLQX2D7KIxSy8nGw==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-port-cli": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/get-port-cli/-/get-port-cli-3.0.0.tgz", + "integrity": "sha512-060GMr81KapTzSobWNrQVAqHeUaFRZhPj/lNnzdCcfVodFN497wRgEamnTCNgldJuiR6TXxdtkFidcYQ/nSVDA==", + "license": "MIT", + "dependencies": { + "get-port": "^6.0.0", + "meow": "^10.1.1" + }, + "bin": { + "get-port": "cli.js" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/get-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", @@ -303,6 +480,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/hard-rejection": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/hard-rejection/-/hard-rejection-2.1.0.tgz", + "integrity": "sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -345,6 +531,18 @@ "he": "bin/he" } }, + "node_modules/hosted-git-info": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-4.1.0.tgz", + "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", + "license": "ISC", + "dependencies": { + "lru-cache": "^6.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/html-encoding-sniffer": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz", @@ -410,6 +608,114 @@ "node": ">=0.10.0" } }, + "node_modules/indent-string": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-5.0.0.tgz", + "integrity": "sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "license": "MIT" + }, + "node_modules/is-core-module": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", + "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", + "license": "MIT", + "dependencies": { + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-plain-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", + "integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "license": "MIT" + }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "license": "MIT" + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/map-obj": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-4.3.0.tgz", + "integrity": "sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -419,6 +725,32 @@ "node": ">= 0.4" } }, + "node_modules/meow": { + "version": "10.1.5", + "resolved": "https://registry.npmjs.org/meow/-/meow-10.1.5.tgz", + "integrity": "sha512-/d+PQ4GKmGvM9Bee/DPa8z3mXs/pkvJE2KEThngVNOqtmljC6K7NMPxtc2JeZYTmpWb9k/TmxjeL18ez3h7vCw==", + "license": "MIT", + "dependencies": { + "@types/minimist": "^1.2.2", + "camelcase-keys": "^7.0.0", + "decamelize": "^5.0.0", + "decamelize-keys": "^1.1.0", + "hard-rejection": "^2.1.0", + "minimist-options": "4.1.0", + "normalize-package-data": "^3.0.2", + "read-pkg-up": "^8.0.0", + "redent": "^4.0.0", + "trim-newlines": "^4.0.2", + "type-fest": "^1.2.2", + "yargs-parser": "^20.2.9" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/mime": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", @@ -440,12 +772,41 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minimist-options": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/minimist-options/-/minimist-options-4.1.0.tgz", + "integrity": "sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==", + "license": "MIT", + "dependencies": { + "arrify": "^1.0.1", + "is-plain-obj": "^1.1.0", + "kind-of": "^6.0.3" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/normalize-package-data": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-3.0.3.tgz", + "integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==", + "license": "BSD-2-Clause", + "dependencies": { + "hosted-git-info": "^4.0.1", + "is-core-module": "^2.5.0", + "semver": "^7.3.4", + "validate-npm-package-license": "^3.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/object-inspect": { "version": "1.13.4", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", @@ -467,13 +828,76 @@ "opener": "bin/opener-bin.js" } }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, "node_modules/playwright": { - "version": "1.54.1", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.54.1.tgz", - "integrity": "sha512-peWpSwIBmSLi6aW2auvrUtf2DqY16YYcCMO8rTVx486jKmDTJg7UAhyrraP98GB8BoPURZP8+nxO7TSd4cPr5g==", + "version": "1.58.2", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.2.tgz", + "integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==", "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.54.1" + "playwright-core": "1.58.2" }, "bin": { "playwright": "cli.js" @@ -486,9 +910,9 @@ } }, "node_modules/playwright-core": { - "version": "1.54.1", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.54.1.tgz", - "integrity": "sha512-Nbjs2zjj0htNhzgiy5wu+3w09YetDx5pkrpI/kZotDlDUaYk0HVA5xrBVPdow4SAUIlhgKcJeJg4GRKW6xHusA==", + "version": "1.58.2", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.2.tgz", + "integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==", "license": "Apache-2.0", "bin": { "playwright-core": "cli.js" @@ -511,9 +935,9 @@ } }, "node_modules/qs": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", - "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.0.tgz", + "integrity": "sha512-mAZTtNCeetKMH+pSjrb76NAM8V9a05I9aBZOHztWy/UqcJdQYNsf59vrRKWnojAT9Y+GbIvoTBC++CPHqpDBhQ==", "license": "BSD-3-Clause", "dependencies": { "side-channel": "^1.1.0" @@ -525,6 +949,69 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/quick-lru": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", + "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/read-pkg": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-6.0.0.tgz", + "integrity": "sha512-X1Fu3dPuk/8ZLsMhEj5f4wFAF0DWoK7qhGJvgaijocXxBmSToKfbFtqbxMO7bVjNA1dmE5huAzjXj/ey86iw9Q==", + "license": "MIT", + "dependencies": { + "@types/normalize-package-data": "^2.4.0", + "normalize-package-data": "^3.0.2", + "parse-json": "^5.2.0", + "type-fest": "^1.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/read-pkg-up": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-8.0.0.tgz", + "integrity": "sha512-snVCqPczksT0HS2EC+SxUndvSzn6LRCwpfSvLrIfR5BKDQQZMaI6jPRC9dYvYFDRAuFEAnkwww8kBBNE/3VvzQ==", + "license": "MIT", + "dependencies": { + "find-up": "^5.0.0", + "read-pkg": "^6.0.0", + "type-fest": "^1.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/redent": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/redent/-/redent-4.0.0.tgz", + "integrity": "sha512-tYkDkVVtYkSVhuQ4zBgfvciymHaeuel+zFKXShfDnFP5SyVEP7qo70Rf1jTOTCx3vGNAbnEi/xFkcfQVMIBWag==", + "license": "MIT", + "dependencies": { + "indent-string": "^5.0.0", + "strip-indent": "^4.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/requires-port": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", @@ -549,6 +1036,18 @@ "integrity": "sha512-AckIIV90rPDcBcglUwXPF3kg0P0qmPsPXAj6BBEENQE1p5yA1xfmDJzfi1Tappj37Pv2mVbKpL3Z1T+Nn7k1Qw==", "license": "MIT" }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/side-channel": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", @@ -621,6 +1120,50 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/spdx-correct": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.2.0.tgz", + "integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==", + "license": "Apache-2.0", + "dependencies": { + "spdx-expression-parse": "^3.0.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/spdx-exceptions": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz", + "integrity": "sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==", + "license": "CC-BY-3.0" + }, + "node_modules/spdx-expression-parse": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz", + "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", + "license": "MIT", + "dependencies": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/spdx-license-ids": { + "version": "3.0.23", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.23.tgz", + "integrity": "sha512-CWLcCCH7VLu13TgOH+r8p1O/Znwhqv/dbb6lqWy67G+pT1kHmeD/+V36AVb/vq8QMIQwVShJ6Ssl5FPh0fuSdw==", + "license": "CC0-1.0" + }, + "node_modules/strip-indent": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-4.1.1.tgz", + "integrity": "sha512-SlyRoSkdh1dYP0PzclLE7r0M9sgbFKKMFXpFRUMNuKhQSbC6VQIGzq3E0qsfvGJaUFJPGv6Ws1NZ/haTAjfbMA==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -633,10 +1176,34 @@ "node": ">=8" } }, + "node_modules/trim-newlines": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-4.1.1.tgz", + "integrity": "sha512-jRKj0n0jXWo6kh62nA5TEh3+4igKDXLvzBJcPpiizP7oOolUrYIxmVBG9TOtHYFHoddUk6YvAkGeGoSVTXfQXQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/type-fest": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-1.4.0.tgz", + "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/undici-types": { - "version": "7.8.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.8.0.tgz", - "integrity": "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", "license": "MIT" }, "node_modules/union": { @@ -656,6 +1223,16 @@ "integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==", "license": "MIT" }, + "node_modules/validate-npm-package-license": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", + "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", + "license": "Apache-2.0", + "dependencies": { + "spdx-correct": "^3.0.0", + "spdx-expression-parse": "^3.0.0" + } + }, "node_modules/whatwg-encoding": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz", @@ -667,6 +1244,33 @@ "engines": { "node": ">=12" } + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "license": "ISC" + }, + "node_modules/yargs-parser": { + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } } } } diff --git a/Platforms/emscripten/browser_test/package.json b/Platforms/emscripten/browser_test/package.json index 3320d4cccd0594..540c9b8034e7c7 100644 --- a/Platforms/emscripten/browser_test/package.json +++ b/Platforms/emscripten/browser_test/package.json @@ -11,7 +11,8 @@ "description": "", "dependencies": { "@playwright/test": "^1.54.1", - "@types/node": "^24.1.0", + "@types/node": "^24.12.0", + "get-port-cli": "^3.0.0", "http-server": "^14.1.1", "playwright": "^1.54.1" } diff --git a/Platforms/emscripten/browser_test/playwright.config.ts b/Platforms/emscripten/browser_test/playwright.config.ts index 0b38beb12826a9..d170789a5970ec 100644 --- a/Platforms/emscripten/browser_test/playwright.config.ts +++ b/Platforms/emscripten/browser_test/playwright.config.ts @@ -1,4 +1,8 @@ import { defineConfig, devices } from '@playwright/test'; +import { resolve } from "node:path"; + +const port = process.env.PORT ?? "8787"; +const crossBuildDir = resolve("../../../", process.env.CROSS_BUILD_DIR ?? "cross-build"); export default defineConfig({ testDir: '.', @@ -6,7 +10,7 @@ export default defineConfig({ retries: 2, reporter: process.env.CI ? 'dot' : 'html', use: { - baseURL: 'http://localhost:8787', + baseURL: `http://localhost:${port}`, trace: 'on-first-retry', }, projects: [ @@ -16,7 +20,7 @@ export default defineConfig({ }, ], webServer: { - command: 'npx http-server ../../../cross-build/wasm32-emscripten/build/python/web_example_pyrepl_jspi/ -p 8787', - url: 'http://localhost:8787', + command: `npx http-server ${crossBuildDir}/wasm32-emscripten/build/python/web_example_pyrepl_jspi/ -p ${port}`, + url: `http://localhost:${port}`, }, }); diff --git a/Platforms/emscripten/browser_test/run_test.sh b/Platforms/emscripten/browser_test/run_test.sh index 9166e0d740585e..cc89b3a91607ed 100755 --- a/Platforms/emscripten/browser_test/run_test.sh +++ b/Platforms/emscripten/browser_test/run_test.sh @@ -6,5 +6,6 @@ echo "Installing node packages" | tee test_log.txt npm ci >> test_log.txt 2>&1 echo "Installing playwright browsers" | tee test_log.txt npx playwright install 2>> test_log.txt -echo "Running tests" | tee test_log.txt +export PORT=$(npx get-port-cli) +echo "Running tests with webserver on port $PORT" | tee test_log.txt CI=1 npx playwright test | tee test_log.txt diff --git a/Platforms/emscripten/browser_test/tsconfig.json b/Platforms/emscripten/browser_test/tsconfig.json new file mode 100644 index 00000000000000..29a2d833656b53 --- /dev/null +++ b/Platforms/emscripten/browser_test/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "esnext", + "module": "nodenext", + "lib": ["ES2020"], + "strict": true, + "esModuleInterop": true, + "types": ["node"] + }, + "include": ["**/*.ts"], + "exclude": ["node_modules"] +} diff --git a/Platforms/emscripten/config.toml b/Platforms/emscripten/config.toml index c474078fb48ba3..ba2dc8f4a482bf 100644 --- a/Platforms/emscripten/config.toml +++ b/Platforms/emscripten/config.toml @@ -11,13 +11,16 @@ test-args = [ "--single-process", "-W", ] +pythoninfo-args = [ + "-m", "test.pythoninfo", +] -[libffi] +[dependencies.libffi] url = "https://github.com/libffi/libffi/releases/download/v{version}/libffi-{version}.tar.gz" version = "3.4.6" shasum = "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e" -[mpdec] +[dependencies.mpdec] url = "https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{version}.tar.gz" version = "4.0.1" shasum = "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8" diff --git a/Platforms/emscripten/node_entry.mjs b/Platforms/emscripten/node_entry.mjs index 166df40742b7fc..110aadc5de1014 100644 --- a/Platforms/emscripten/node_entry.mjs +++ b/Platforms/emscripten/node_entry.mjs @@ -1,5 +1,6 @@ import EmscriptenModule from "./python.mjs"; import fs from "node:fs"; +import { initializeStreams } from "./streams.mjs"; if (process?.versions?.node) { const nodeVersion = Number(process.versions.node.split(".", 1)[0]); @@ -39,6 +40,9 @@ const settings = { Object.assign(Module.ENV, process.env); delete Module.ENV.PATH; }, + onRuntimeInitialized() { + initializeStreams(Module.FS); + }, // Ensure that sys.executable, sys._base_executable, etc point to python.sh // not to this file. To properly handle symlinks, python.sh needs to compute // its own path. @@ -49,10 +53,10 @@ const settings = { try { await EmscriptenModule(settings); -} catch(e) { +} catch (e) { // Show JavaScript exception and traceback console.warn(e); // Show Python exception and traceback - Module.__Py_DumpTraceback(2, Module._PyGILState_GetThisThreadState()); + Module.PyUnstable_DumpTraceback(2, Module._PyGILState_GetThisThreadState()); process.exit(1); } diff --git a/Platforms/emscripten/streams.mjs b/Platforms/emscripten/streams.mjs new file mode 100644 index 00000000000000..76ad79f9247f4c --- /dev/null +++ b/Platforms/emscripten/streams.mjs @@ -0,0 +1,241 @@ +/** + * This is a pared down version of + * https://github.com/pyodide/pyodide/blob/main/src/js/streams.ts + * + * It replaces the standard streams devices that Emscripten provides with our + * own better ones. It fixes the following deficiencies: + * + * 1. The emscripten std streams always have isatty set to true. These set + * isatty to match the value for the stdin/stdout/stderr that node sees. + * 2. The emscripten std streams don't support the ttygetwinsize ioctl. If + * isatty() returns true, then these do, and it returns the actual window + * size as the OS reports it to Node. + * 3. The emscripten std streams introduce an extra layer of buffering which has + * to be flushed with fsync(). + * 4. The emscripten std streams are slow and complex because they go through a + * character-based handler layer. This is particularly awkward because both + * sides of this character based layer deal with buffers and so we need + * complex adaptors, buffering, etc on both sides. Removing this + * character-based middle layer makes everything better. + * https://github.com/emscripten-core/emscripten/blob/1aa7fb531f11e11e7ae49b75a24e1a8fe6fa4a7d/src/lib/libtty.js?plain=1#L104-L114 + * + * Ideally some version of this should go upstream to Emscripten since it is not + * in any way specific to Python. But I (Hood) haven't gotten around to it yet. + */ + +import * as tty from "node:tty"; +import * as fs from "node:fs"; + +let FS; +const DEVOPS = {}; +const DEVS = {}; + +function isErrnoError(e) { + return e && typeof e === "object" && "errno" in e; +} + +const waitBuffer = new Int32Array( + new WebAssembly.Memory({ shared: true, initial: 1, maximum: 1 }).buffer, +); +function syncSleep(timeout) { + try { + Atomics.wait(waitBuffer, 0, 0, timeout); + return true; + } catch (_) { + return false; + } +} + +/** + * Calls the callback and handle node EAGAIN errors. + */ +function handleEAGAIN(cb) { + while (true) { + try { + return cb(); + } catch (e) { + if (e && e.code === "EAGAIN") { + // Presumably this means we're in node and tried to read from/write to + // an O_NONBLOCK file descriptor. Synchronously sleep for 10ms then try + // again. In case for some reason we fail to sleep, propagate the error + // (it will turn into an EOFError). + if (syncSleep(10)) { + continue; + } + } + throw e; + } + } +} + +function readWriteHelper(stream, cb, method) { + let nbytes; + try { + nbytes = handleEAGAIN(cb); + } catch (e) { + if (e && e.code && Module.ERRNO_CODES[e.code]) { + throw new FS.ErrnoError(Module.ERRNO_CODES[e.code]); + } + if (isErrnoError(e)) { + // the handler set an errno, propagate it + throw e; + } + console.error("Error thrown in read:"); + console.error(e); + throw new FS.ErrnoError(Module.ERRNO_CODES.EIO); + } + if (nbytes === undefined) { + // Prevent an infinite loop caused by incorrect code that doesn't return a + // value. + // Maybe we should set nbytes = buffer.length here instead? + console.warn( + `${method} returned undefined; a correct implementation must return a number`, + ); + throw new FS.ErrnoError(Module.ERRNO_CODES.EIO); + } + if (nbytes !== 0) { + stream.node.timestamp = Date.now(); + } + return nbytes; +} + +function asUint8Array(arg) { + if (ArrayBuffer.isView(arg)) { + return new Uint8Array(arg.buffer, arg.byteOffset, arg.byteLength); + } else { + return new Uint8Array(arg); + } +} + +const prepareBuffer = (buffer, offset, length) => + asUint8Array(buffer).subarray(offset, offset + length); + +const TTY_OPS = { + ioctl_tiocgwinsz(tty) { + return tty.devops.ioctl_tiocgwinsz?.(); + }, +}; + +const stream_ops = { + open: function (stream) { + const devops = DEVOPS[stream.node.rdev]; + if (!devops) { + throw new FS.ErrnoError(Module.ERRNO_CODES.ENODEV); + } + stream.devops = devops; + stream.tty = stream.devops.isatty ? { ops: TTY_OPS, devops } : undefined; + stream.seekable = false; + }, + close: function (stream) { + // flush any pending line data + stream.stream_ops.fsync(stream); + }, + fsync: function (stream) { + const ops = stream.devops; + if (ops.fsync) { + ops.fsync(); + } + }, + read: function (stream, buffer, offset, length, pos /* ignored */) { + buffer = prepareBuffer(buffer, offset, length); + return readWriteHelper(stream, () => stream.devops.read(buffer), "read"); + }, + write: function (stream, buffer, offset, length, pos /* ignored */) { + buffer = prepareBuffer(buffer, offset, length); + return readWriteHelper(stream, () => stream.devops.write(buffer), "write"); + }, +}; + +function nodeFsync(fd) { + try { + fs.fsyncSync(fd); + } catch (e) { + if (e?.code === "EINVAL") { + return; + } + // On Mac, calling fsync when not isatty returns ENOTSUP + // On Windows, stdin/stdout/stderr may be closed, returning EBADF or EPERM + if ( + e?.code === "ENOTSUP" || e?.code === "EBADF" || e?.code === "EPERM" + ) { + return; + } + + throw e; + } +} + +class NodeReader { + constructor(nodeStream) { + this.nodeStream = nodeStream; + this.isatty = tty.isatty(nodeStream.fd); + } + + read(buffer) { + try { + return fs.readSync(this.nodeStream.fd, buffer); + } catch (e) { + // Platform differences: on Windows, reading EOF throws an exception, + // but on other OSes, reading EOF returns 0. Uniformize behavior by + // catching the EOF exception and returning 0. + if (e.toString().includes("EOF")) { + return 0; + } + throw e; + } + } + + fsync() { + nodeFsync(this.nodeStream.fd); + } +} + +class NodeWriter { + constructor(nodeStream) { + this.nodeStream = nodeStream; + this.isatty = tty.isatty(nodeStream.fd); + } + + write(buffer) { + return fs.writeSync(this.nodeStream.fd, buffer); + } + + fsync() { + nodeFsync(this.nodeStream.fd); + } + + ioctl_tiocgwinsz() { + return [this.nodeStream.rows ?? 24, this.nodeStream.columns ?? 80]; + } +} + +export function initializeStreams(fsarg) { + FS = fsarg; + const major = FS.createDevice.major++; + DEVS.stdin = FS.makedev(major, 0); + DEVS.stdout = FS.makedev(major, 1); + DEVS.stderr = FS.makedev(major, 2); + + FS.registerDevice(DEVS.stdin, stream_ops); + FS.registerDevice(DEVS.stdout, stream_ops); + FS.registerDevice(DEVS.stderr, stream_ops); + + FS.unlink("/dev/stdin"); + FS.unlink("/dev/stdout"); + FS.unlink("/dev/stderr"); + + FS.mkdev("/dev/stdin", DEVS.stdin); + FS.mkdev("/dev/stdout", DEVS.stdout); + FS.mkdev("/dev/stderr", DEVS.stderr); + + DEVOPS[DEVS.stdin] = new NodeReader(process.stdin); + DEVOPS[DEVS.stdout] = new NodeWriter(process.stdout); + DEVOPS[DEVS.stderr] = new NodeWriter(process.stderr); + + FS.closeStream(0 /* stdin */); + FS.closeStream(1 /* stdout */); + FS.closeStream(2 /* stderr */); + FS.open("/dev/stdin", 0 /* O_RDONLY */); + FS.open("/dev/stdout", 1 /* O_WRONLY */); + FS.open("/dev/stderr", 1 /* O_WRONLY */); +} diff --git a/Platforms/emscripten/web_example_pyrepl_jspi/src.mjs b/Platforms/emscripten/web_example_pyrepl_jspi/src.mjs index 5642372c9d2472..38a622117c2a50 100644 --- a/Platforms/emscripten/web_example_pyrepl_jspi/src.mjs +++ b/Platforms/emscripten/web_example_pyrepl_jspi/src.mjs @@ -189,6 +189,6 @@ try { // Show JavaScript exception and traceback console.warn(e); // Show Python exception and traceback - Module.__Py_DumpTraceback(2, Module._PyGILState_GetThisThreadState()); + Module.PyUnstable_DumpTraceback(2, Module._PyGILState_GetThisThreadState()); process.exit(1); } diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 1411eb1718b683..d550740b1105dd 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,19 +1,19 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0, - 0,0,0,0,0,243,186,0,0,0,128,0,0,0,94,0, - 82,1,73,0,116,0,94,0,82,1,73,4,116,1,93,2, - 33,0,82,2,52,1,0,0,0,0,0,0,31,0,93,2, - 33,0,82,3,93,0,80,6,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,52,2,0,0,0,0, - 0,0,31,0,93,1,80,8,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,33,0,52,0,0,0, - 0,0,0,0,82,4,44,26,0,0,0,0,0,0,0,0, - 0,0,116,5,82,7,16,0,70,24,0,0,116,6,93,2, - 33,0,82,5,93,6,12,0,82,6,93,5,93,6,44,26, - 0,0,0,0,0,0,0,0,0,0,12,0,50,4,52,1, - 0,0,0,0,0,0,31,0,75,26,0,0,9,0,30,0, - 82,1,35,0,41,8,233,0,0,0,0,78,122,18,70,114, + 0,0,0,0,0,243,186,0,0,0,128,0,0,0,93,0, + 81,1,72,0,115,0,93,0,81,1,72,4,115,1,92,2, + 31,0,81,2,50,1,0,0,0,0,0,0,29,0,92,2, + 31,0,81,3,92,0,79,6,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,50,2,0,0,0,0, + 0,0,29,0,92,1,79,8,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,31,0,50,0,0,0, + 0,0,0,0,81,4,42,26,0,0,0,0,0,0,0,0, + 0,0,115,5,81,7,70,0,68,24,0,0,115,6,92,2, + 31,0,81,5,92,6,12,0,81,6,92,5,92,6,42,26, + 0,0,0,0,0,0,0,0,0,0,12,0,48,4,50,1, + 0,0,0,0,0,0,29,0,74,26,0,0,9,0,28,0, + 81,1,33,0,41,8,233,0,0,0,0,78,122,18,70,114, 111,122,101,110,32,72,101,108,108,111,32,87,111,114,108,100, 122,8,115,121,115,46,97,114,103,118,218,6,99,111,110,102, 105,103,122,7,99,111,110,102,105,103,32,122,2,58,32,41, diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 5d319992dcda1e..dad1530e343a38 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -18480,6 +18480,7 @@ astmodule_exec(PyObject *m) } static PyModuleDef_Slot astmodule_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, astmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index c50ff1190686c2..e6d39e4c7dc823 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -400,6 +400,7 @@ static PyMethodDef tokenize_methods[] = { }; static PyModuleDef_Slot tokenizemodule_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, tokenizemodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Python/_contextvars.c b/Python/_contextvars.c index 0f8b8004c1af22..86acc94fbc79cd 100644 --- a/Python/_contextvars.c +++ b/Python/_contextvars.c @@ -43,6 +43,7 @@ _contextvars_exec(PyObject *m) } static struct PyModuleDef_Slot _contextvars_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, _contextvars_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Python/_warnings.c b/Python/_warnings.c index 6b6ac238935765..6b8fa19ff3f606 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1626,6 +1626,7 @@ warnings_module_exec(PyObject *module) static PyModuleDef_Slot warnings_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, warnings_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Python/assemble.c b/Python/assemble.c index 7c08488092de63..3df959c3634195 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -418,6 +418,7 @@ assemble_emit_instr(struct assembler *a, instruction *instr) int size = instr_size(instr); if (a->a_offset + size >= len / (int)sizeof(_Py_CODEUNIT)) { if (len > PY_SSIZE_T_MAX / 2) { + PyErr_NoMemory(); return ERROR; } RETURN_IF_ERROR(_PyBytes_Resize(&a->a_bytecode, len * 2)); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 5680e8971576cd..fec64e1ff9d25f 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1607,7 +1607,7 @@ map_next(PyObject *self) // ValueError: map() argument 3 is shorter than arguments 1-2 const char* plural = i == 1 ? " " : "s 1-"; PyErr_Format(PyExc_ValueError, - "map() argument %d is shorter than argument%s%d", + "map() argument %zd is shorter than argument%s%zd", i + 1, plural, i); goto exit_no_result; } @@ -1618,7 +1618,7 @@ map_next(PyObject *self) Py_DECREF(val); const char* plural = i == 1 ? " " : "s 1-"; PyErr_Format(PyExc_ValueError, - "map() argument %d is longer than argument%s%d", + "map() argument %zd is longer than argument%s%zd", i + 1, plural, i); goto exit_no_result; } @@ -3307,7 +3307,7 @@ zip_next(PyObject *self) // ValueError: zip() argument 3 is shorter than arguments 1-2 const char* plural = i == 1 ? " " : "s 1-"; return PyErr_Format(PyExc_ValueError, - "zip() argument %d is shorter than argument%s%d", + "zip() argument %zd is shorter than argument%s%zd", i + 1, plural, i); } for (i = 1; i < tuplesize; i++) { @@ -3317,7 +3317,7 @@ zip_next(PyObject *self) Py_DECREF(item); const char* plural = i == 1 ? " " : "s 1-"; return PyErr_Format(PyExc_ValueError, - "zip() argument %d is longer than argument%s%d", + "zip() argument %zd is longer than argument%s%zd", i + 1, plural, i); } if (PyErr_Occurred()) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index edccb1ea5a0144..edba5a89cc0f29 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -33,7 +33,7 @@ #include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs #include "pycore_stackref.h" #include "pycore_template.h" // _PyTemplate_Build() -#include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_tuple.h" // _PyStolenTuple_Free(), _PyTuple_ITEMS() #include "pycore_typeobject.h" // _PySuper_Lookup() #include "pycore_dict.h" @@ -425,13 +425,15 @@ dummy_func( PyStackRef_CLOSE(iter); } - pure inst(END_SEND, (receiver, value -- val)) { + pure inst(END_SEND, (receiver, index_or_null, value -- val)) { val = value; + (void)index_or_null; DEAD(value); + DEAD(index_or_null); PyStackRef_CLOSE(receiver); } - tier1 inst(INSTRUMENTED_END_SEND, (receiver, value -- val)) { + tier1 inst(INSTRUMENTED_END_SEND, (receiver, index_or_null, value -- val)) { PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); @@ -440,6 +442,8 @@ dummy_func( } } val = value; + (void)index_or_null; + DEAD(index_or_null); DEAD(value); PyStackRef_CLOSE(receiver); } @@ -456,6 +460,20 @@ dummy_func( DEAD(value); } + // Inplace negation: negate a uniquely-referenced float in place. + // Tier 2 only. + tier2 op(_UNARY_NEGATIVE_FLOAT_INPLACE, (value -- res, v)) { + PyObject *val_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyFloat_CheckExact(val_o)); + assert(_PyObject_IsUniquelyReferenced(val_o)); + STAT_INC(UNARY_NEGATIVE, hit); + double dres = -((PyFloatObject *)val_o)->ob_fval; + ((PyFloatObject *)val_o)->ob_fval = dres; + res = value; + v = PyStackRef_NULL; + INPUTS_DEAD(); + } + inst(UNARY_NOT, (value -- res)) { assert(PyStackRef_BoolCheck(value)); res = PyStackRef_IsFalse(value) @@ -690,6 +708,63 @@ dummy_func( macro(BINARY_OP_SUBTRACT_INT) = _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_SUBTRACT_INT + _POP_TOP_INT + _POP_TOP_INT; + // Inplace compact int ops: mutate the uniquely-referenced operand + // when possible. The op handles decref of TARGET internally so + // the following _POP_TOP_INT becomes _POP_TOP_NOP. Tier 2 only. + tier2 op(_BINARY_OP_ADD_INT_INPLACE, (left, right -- res, l, r)) { + INT_INPLACE_OP(left, right, left, +, _PyCompactLong_Add); + EXIT_IF(PyStackRef_IsNull(_int_inplace_res)); + res = _int_inplace_res; + l = left; + r = right; + INPUTS_DEAD(); + } + + tier2 op(_BINARY_OP_SUBTRACT_INT_INPLACE, (left, right -- res, l, r)) { + INT_INPLACE_OP(left, right, left, -, _PyCompactLong_Subtract); + EXIT_IF(PyStackRef_IsNull(_int_inplace_res)); + res = _int_inplace_res; + l = left; + r = right; + INPUTS_DEAD(); + } + + tier2 op(_BINARY_OP_MULTIPLY_INT_INPLACE, (left, right -- res, l, r)) { + INT_INPLACE_OP(left, right, left, *, _PyCompactLong_Multiply); + EXIT_IF(PyStackRef_IsNull(_int_inplace_res)); + res = _int_inplace_res; + l = left; + r = right; + INPUTS_DEAD(); + } + + tier2 op(_BINARY_OP_ADD_INT_INPLACE_RIGHT, (left, right -- res, l, r)) { + INT_INPLACE_OP(left, right, right, +, _PyCompactLong_Add); + EXIT_IF(PyStackRef_IsNull(_int_inplace_res)); + res = _int_inplace_res; + l = left; + r = right; + INPUTS_DEAD(); + } + + tier2 op(_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, (left, right -- res, l, r)) { + INT_INPLACE_OP(left, right, right, -, _PyCompactLong_Subtract); + EXIT_IF(PyStackRef_IsNull(_int_inplace_res)); + res = _int_inplace_res; + l = left; + r = right; + INPUTS_DEAD(); + } + + tier2 op(_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, (left, right -- res, l, r)) { + INT_INPLACE_OP(left, right, right, *, _PyCompactLong_Multiply); + EXIT_IF(PyStackRef_IsNull(_int_inplace_res)); + res = _int_inplace_res; + l = left; + r = right; + INPUTS_DEAD(); + } + op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); EXIT_IF(!PyFloat_CheckExact(left_o)); @@ -767,6 +842,59 @@ dummy_func( macro(BINARY_OP_SUBTRACT_FLOAT) = _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_SUBTRACT_FLOAT + _POP_TOP_FLOAT + _POP_TOP_FLOAT; + // Inplace float ops: mutate the uniquely-referenced left operand + // instead of allocating a new float. Tier 2 only. + // The optimizer sets l to a borrowed value so the following _POP_TOP_FLOAT + // becomes _POP_TOP_NOP. + tier2 op(_BINARY_OP_ADD_FLOAT_INPLACE, (left, right -- res, l, r)) { + FLOAT_INPLACE_OP(left, right, left, +); + res = left; + l = PyStackRef_NULL; + r = right; + INPUTS_DEAD(); + } + + tier2 op(_BINARY_OP_SUBTRACT_FLOAT_INPLACE, (left, right -- res, l, r)) { + FLOAT_INPLACE_OP(left, right, left, -); + res = left; + l = PyStackRef_NULL; + r = right; + INPUTS_DEAD(); + } + + tier2 op(_BINARY_OP_MULTIPLY_FLOAT_INPLACE, (left, right -- res, l, r)) { + FLOAT_INPLACE_OP(left, right, left, *); + res = left; + l = PyStackRef_NULL; + r = right; + INPUTS_DEAD(); + } + + // Inplace RIGHT variants: mutate the uniquely-referenced right operand. + tier2 op(_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT, (left, right -- res, l, r)) { + FLOAT_INPLACE_OP(left, right, right, +); + res = right; + l = left; + r = PyStackRef_NULL; + INPUTS_DEAD(); + } + + tier2 op(_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT, (left, right -- res, l, r)) { + FLOAT_INPLACE_OP(left, right, right, *); + res = right; + l = left; + r = PyStackRef_NULL; + INPUTS_DEAD(); + } + + tier2 op(_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, (left, right -- res, l, r)) { + FLOAT_INPLACE_OP(left, right, right, -); + res = right; + l = left; + r = PyStackRef_NULL; + INPUTS_DEAD(); + } + pure op(_BINARY_OP_ADD_UNICODE, (left, right -- res, l, r)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -1104,6 +1232,26 @@ dummy_func( macro(BINARY_OP_SUBSCR_DICT) = _GUARD_NOS_ANY_DICT + unused/5 + _BINARY_OP_SUBSCR_DICT + POP_TOP + POP_TOP; + tier2 op(_BINARY_OP_SUBSCR_DICT_KNOWN_HASH, (dict_st, sub_st, hash/4 -- res, ds, ss)) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + + assert(PyAnyDict_CheckExact(dict)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o; + int rc = _PyDict_GetItemRef_KnownHash((PyDictObject *)dict, sub, (Py_hash_t)hash, &res_o); + if (rc == 0) { + _PyErr_SetKeyError(sub); + } + if (rc <= 0) { + ERROR_NO_POP(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + ds = dict_st; + ss = sub_st; + INPUTS_DEAD(); + } + op(_BINARY_OP_SUBSCR_DICT, (dict_st, sub_st -- res, ds, ss)) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); @@ -1248,6 +1396,24 @@ dummy_func( st = dict_st; } + tier2 op(_STORE_SUBSCR_DICT_KNOWN_HASH, (value, dict_st, sub, hash/4 -- st)) { + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + + assert(PyDict_CheckExact(dict)); + STAT_INC(STORE_SUBSCR, hit); + int err = _PyDict_SetItem_Take2_KnownHash((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value), + (Py_hash_t)hash); + + if (err) { + PyStackRef_CLOSE(dict_st); + ERROR_IF(1); + } + DEAD(dict_st); + st = dict_st; + } + inst(DELETE_SUBSCR, (container, sub --)) { /* del container[sub] */ int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), @@ -1269,17 +1435,26 @@ dummy_func( macro(CALL_INTRINSIC_1) = _CALL_INTRINSIC_1 + POP_TOP; - inst(CALL_INTRINSIC_2, (value2_st, value1_st -- res)) { + op(_CALL_INTRINSIC_2, (value2_st, value1_st -- res, vs1, vs2)) { assert(oparg <= MAX_INTRINSIC_2); PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - DECREF_INPUTS(); - ERROR_IF(res_o == NULL); + + if (res_o == NULL) { + ERROR_NO_POP(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + + vs1 = value1_st; + vs2 = value2_st; + INPUTS_DEAD(); } + macro(CALL_INTRINSIC_2) = _CALL_INTRINSIC_2 + POP_TOP + POP_TOP; + tier1 inst(RAISE_VARARGS, (args[oparg] -- )) { assert(oparg < 3); PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; @@ -1410,7 +1585,7 @@ dummy_func( SEND_GEN, }; - specializing op(_SPECIALIZE_SEND, (counter/1, receiver, unused -- receiver, unused)) { + specializing op(_SPECIALIZE_SEND, (counter/1, receiver, unused, unused -- receiver, unused, unused)) { #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; @@ -1422,7 +1597,7 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ } - op(_SEND, (receiver, v -- receiver, retval)) { + op(_SEND, (receiver, null_or_index, v -- receiver, null_or_index, retval)) { PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); PyObject *retval_o; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); @@ -1443,36 +1618,49 @@ dummy_func( gen_frame->previous = frame; DISPATCH_INLINED(gen_frame); } - if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { - retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + if (!PyStackRef_IsNull(null_or_index)) { + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, receiver, &null_or_index); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + ERROR_NO_POP(); + } + JUMPBY(oparg); + DISPATCH(); + } + retval = item; } else { - retval_o = PyObject_CallMethodOneArg(receiver_o, - &_Py_ID(send), - PyStackRef_AsPyObjectBorrow(v)); - } - if (retval_o == NULL) { - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - if (matches) { - _PyEval_MonitorRaise(tstate, frame, this_instr); - } - int err = _PyGen_FetchStopIterationValue(&retval_o); - if (err == 0) { - assert(retval_o != NULL); - JUMPBY(oparg); + if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); } else { - PyStackRef_CLOSE(v); - ERROR_IF(true); + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); } + if (retval_o == NULL) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (matches) { + _PyEval_MonitorRaise(tstate, frame, this_instr); + } + int err = _PyGen_FetchStopIterationValue(&retval_o); + if (err == 0) { + assert(retval_o != NULL); + JUMPBY(oparg); + } + else { + PyStackRef_CLOSE(v); + ERROR_IF(true); + } + } + retval = PyStackRef_FromPyObjectSteal(retval_o); } PyStackRef_CLOSE(v); - retval = PyStackRef_FromPyObjectSteal(retval_o); } macro(SEND) = _SPECIALIZE_SEND + _SEND; - op(_SEND_GEN_FRAME, (receiver, v -- receiver, gen_frame)) { + op(_SEND_GEN_FRAME, (receiver, null, v -- receiver, null, gen_frame)) { PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); EXIT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type); EXIT_IF(!gen_try_set_executing((PyGenObject *)gen)); @@ -1490,7 +1678,7 @@ dummy_func( macro(SEND_GEN) = unused/1 + - _RECORD_NOS_GEN_FUNC + + _RECORD_3OS_GEN_FUNC + _CHECK_PEP_523 + _SEND_GEN_FRAME + _PUSH_FRAME; @@ -1602,17 +1790,17 @@ dummy_func( macro(END_ASYNC_FOR) = _END_ASYNC_FOR; - tier1 inst(CLEANUP_THROW, (sub_iter, last_sent_val, exc_value_st -- none, value)) { + tier1 inst(CLEANUP_THROW, (sub_iter, null_in, last_sent_val, exc_value_st -- none, null_out, value)) { PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); #if !_Py_TAIL_CALL_INTERP assert(throwflag); #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); - int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); if (matches) { value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); DECREF_INPUTS(); + null_out = null_in; none = PyStackRef_None; } else { @@ -1722,6 +1910,25 @@ dummy_func( PyStackRef_CLOSE(seq); } + op(_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, (seq -- val1, val0)) { + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + } + + op(_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE, (seq -- val2, val1, val0)) { + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); + val2 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 2)); + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + } + macro(UNPACK_SEQUENCE_TUPLE) = _GUARD_TOS_TUPLE + unused/1 + _UNPACK_SEQUENCE_TUPLE; @@ -1737,6 +1944,20 @@ dummy_func( DECREF_INPUTS(); } + op(_UNPACK_SEQUENCE_UNIQUE_TUPLE, (seq -- values[oparg])) { + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + assert(PyTuple_CheckExact(seq_o)); + assert(PyTuple_GET_SIZE(seq_o) == oparg); + assert(_PyObject_IsUniquelyReferenced(seq_o)); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectSteal(items[i]); + } + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + } + macro(UNPACK_SEQUENCE_LIST) = _GUARD_TOS_LIST + unused/1 + _UNPACK_SEQUENCE_LIST; @@ -2159,7 +2380,7 @@ dummy_func( list = PyStackRef_FromPyObjectStealMortal(list_o); } - inst(LIST_EXTEND, (list_st, unused[oparg-1], iterable_st -- list_st, unused[oparg-1])) { + op(_LIST_EXTEND, (list_st, unused[oparg-1], iterable_st -- list_st, unused[oparg-1], i)) { PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); @@ -2174,13 +2395,15 @@ dummy_func( "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - PyStackRef_CLOSE(iterable_st); - ERROR_IF(true); + ERROR_NO_POP(); } assert(Py_IsNone(none_val)); - PyStackRef_CLOSE(iterable_st); + i = iterable_st; + DEAD(iterable_st); } + macro(LIST_EXTEND) = _LIST_EXTEND + POP_TOP; + op(_SET_UPDATE, (set, unused[oparg-1], iterable -- set, unused[oparg-1], i)) { int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); @@ -2256,7 +2479,7 @@ dummy_func( NOP, }; - inst(DICT_UPDATE, (dict, unused[oparg - 1], update -- dict, unused[oparg - 1])) { + op(_DICT_UPDATE, (dict, unused[oparg - 1], update -- dict, unused[oparg - 1], upd)) { PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); @@ -2264,30 +2487,44 @@ dummy_func( if (err < 0) { int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); if (matches) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update_o)->tp_name); + PyObject *exc = _PyErr_GetRaisedException(tstate); + int has_keys = PyObject_HasAttrWithError(update_o, &_Py_ID(keys)); + if (has_keys == 0) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%T' object is not a mapping", + update_o); + Py_DECREF(exc); + } + else { + _PyErr_ChainExceptions1(exc); + } } - PyStackRef_CLOSE(update); - ERROR_IF(true); + ERROR_NO_POP(); } - PyStackRef_CLOSE(update); + upd = update; + DEAD(update); } - inst(DICT_MERGE, (callable, unused, unused, dict, unused[oparg - 1], update -- callable, unused, unused, dict, unused[oparg - 1])) { + macro(DICT_UPDATE) = _DICT_UPDATE + POP_TOP; + + op(_DICT_MERGE, (callable, unused, unused, dict, unused[oparg - 1], update -- callable, unused, unused, dict, unused[oparg - 1], u)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + PyObject *dupkey = NULL; - int err = _PyDict_MergeEx(dict_o, update_o, 2); + int err = _PyDict_MergeUniq(dict_o, update_o, &dupkey); if (err < 0) { - _PyEval_FormatKwargsError(tstate, callable_o, update_o); - PyStackRef_CLOSE(update); - ERROR_IF(true); + _PyEval_FormatKwargsError(tstate, callable_o, update_o, dupkey); + Py_XDECREF(dupkey); + ERROR_NO_POP(); } - PyStackRef_CLOSE(update); + u = update; + DEAD(update); } + macro(DICT_MERGE) = _DICT_MERGE + POP_TOP; + inst(MAP_ADD, (dict_st, unused[oparg - 1], key, value -- dict_st, unused[oparg - 1])) { PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); assert(PyDict_CheckExact(dict)); @@ -3335,52 +3572,10 @@ dummy_func( #ifdef Py_STATS _Py_GatherStats_GetIter(iterable); #endif - PyTypeObject *tp = PyStackRef_TYPE(iterable); - if (tp == &PyTuple_Type || tp == &PyList_Type) { - /* Leave iterable on stack and pushed tagged 0 */ - iter = iterable; - DEAD(iterable); - index_or_null = PyStackRef_TagInt(0); - } - else { - /* Pop iterable, and push iterator then NULL */ - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - PyStackRef_CLOSE(iterable); - ERROR_IF(iter_o == NULL); - iter = PyStackRef_FromPyObjectSteal(iter_o); - index_or_null = PyStackRef_NULL; - } - } - - inst(GET_YIELD_FROM_ITER, (iterable -- iter)) { - /* before: [obj]; after [getiter(obj)] */ - PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); - if (PyCoro_CheckExact(iterable_o)) { - /* `iterable` is a coroutine */ - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - /* and it is used in a 'yield from' expression of a - regular generator. */ - _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); - ERROR_NO_POP(); - } - iter = iterable; - DEAD(iterable); - } - else if (PyGen_CheckExact(iterable_o)) { - iter = iterable; - DEAD(iterable); - } - else { - /* `iterable` is not a generator. */ - PyObject *iter_o = PyObject_GetIter(iterable_o); - if (iter_o == NULL) { - ERROR_NO_POP(); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - DECREF_INPUTS(); - } + _PyStackRef result = _PyEval_GetIter(iterable, &index_or_null, oparg); + DEAD(iterable); + ERROR_IF(PyStackRef_IsError(result)); + iter = result; } // Most members of this family are "secretly" super-instructions. @@ -4383,18 +4578,24 @@ dummy_func( _CALL_BUILTIN_CLASS + _CHECK_PERIODIC_AT_END; + op(_GUARD_CALLABLE_BUILTIN_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + EXIT_IF(!PyCFunction_CheckExact(callable_o)); + EXIT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + EXIT_IF(total_args != 1); + } + op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res, c, s)) { /* Builtin METH_O functions */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; if (!PyStackRef_IsNull(self_or_null)) { args--; - total_args++; } - EXIT_IF(total_args != 1); - EXIT_IF(!PyCFunction_CheckExact(callable_o)); - EXIT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O); // CPython promises to check all non-vectorcall function calls. EXIT_IF(_Py_ReachedRecursionLimit(tstate)); STAT_INC(CALL, hit); @@ -4416,11 +4617,18 @@ dummy_func( _RECORD_CALLABLE + unused/1 + unused/2 + + _GUARD_CALLABLE_BUILTIN_O + _CALL_BUILTIN_O + POP_TOP + POP_TOP + _CHECK_PERIODIC_AT_END; + op(_GUARD_CALLABLE_BUILTIN_FAST, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + EXIT_IF(!PyCFunction_CheckExact(callable_o)); + EXIT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL); + } + op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL functions, without keywords */ int total_args = oparg; @@ -4429,9 +4637,6 @@ dummy_func( arguments--; total_args++; } - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - EXIT_IF(!PyCFunction_CheckExact(callable_o)); - EXIT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL); STAT_INC(CALL, hit); PyObject *res_o = _Py_BuiltinCallFast_StackRefSteal( callable, @@ -4449,9 +4654,16 @@ dummy_func( _RECORD_CALLABLE + unused/1 + unused/2 + + _GUARD_CALLABLE_BUILTIN_FAST + _CALL_BUILTIN_FAST + _CHECK_PERIODIC_AT_END; + op(_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + EXIT_IF(!PyCFunction_CheckExact(callable_o)); + EXIT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)); + } + op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int total_args = oparg; @@ -4460,9 +4672,6 @@ dummy_func( arguments--; total_args++; } - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - EXIT_IF(!PyCFunction_CheckExact(callable_o)); - EXIT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)); STAT_INC(CALL, hit); PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRefSteal(callable, arguments, total_args); DEAD(args); @@ -4476,6 +4685,7 @@ dummy_func( _RECORD_CALLABLE + unused/1 + unused/2 + + _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS + _CALL_BUILTIN_FAST_WITH_KEYWORDS + _CHECK_PERIODIC_AT_END; @@ -4578,32 +4788,34 @@ dummy_func( none = PyStackRef_None; } - op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res, c, s, a)) { + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); + EXIT_IF(method->d_method->ml_flags != METH_O); int total_args = oparg; - _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { - arguments--; total_args++; } + EXIT_IF(total_args != 2); + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); + EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); + } + op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res, c, s, a)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - EXIT_IF(total_args != 2); - EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); - PyMethodDef *meth = method->d_method; - EXIT_IF(meth->ml_flags != METH_O); - // CPython promises to check all non-vectorcall function calls. - EXIT_IF(_Py_ReachedRecursionLimit(tstate)); - _PyStackRef arg_stackref = arguments[1]; - _PyStackRef self_stackref = arguments[0]; - EXIT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type)); + + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + } STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, - PyStackRef_AsPyObjectBorrow(self_stackref), - PyStackRef_AsPyObjectBorrow(arg_stackref)); + PyCFunction cfunc = method->d_method->ml_meth; + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + PyObject *arg = PyStackRef_AsPyObjectBorrow(arguments[1]); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -4616,19 +4828,46 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } + op(_CHECK_RECURSION_LIMIT, ( -- )) { + EXIT_IF(_Py_ReachedRecursionLimit(tstate)); + } + + tier2 op(_CALL_METHOD_DESCRIPTOR_O_INLINE, (callable, args[oparg], cfunc/4 -- res, c, s, a)) { + assert(oparg == 2); + STAT_INC(CALL, hit); + volatile PyCFunction cfunc_v = (PyCFunction)cfunc; + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *arg = PyStackRef_AsPyObjectBorrow(args[1]); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc_v, self, arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + ERROR_NO_POP(); + } + c = callable; + s = args[0]; + a = args[1]; + INPUTS_DEAD(); + res = PyStackRef_FromPyObjectSteal(res_o); + } + macro(CALL_METHOD_DESCRIPTOR_O) = _RECORD_CALLABLE + unused/1 + unused/2 + + _GUARD_CALLABLE_METHOD_DESCRIPTOR_O + + _CHECK_RECURSION_LIMIT + _CALL_METHOD_DESCRIPTOR_O + POP_TOP + POP_TOP + POP_TOP + _CHECK_PERIODIC_AT_END; - op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); + EXIT_IF(method->d_method->ml_flags != (METH_FASTCALL|METH_KEYWORDS)); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { @@ -4636,18 +4875,27 @@ dummy_func( total_args++; } EXIT_IF(total_args == 0); + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); + } + + op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); - PyMethodDef *meth = method->d_method; - EXIT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)); - PyTypeObject *d_type = method->d_common.d_type; + + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); assert(self != NULL); - EXIT_IF(!Py_IS_TYPE(self, d_type)); STAT_INC(CALL, hit); + PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords_CAST(method->d_method->ml_meth); PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( callable, - meth, + cfunc, self, arguments, total_args @@ -4659,34 +4907,59 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } + tier2 op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, (callable, args[oparg], cfunc/4 -- res)) { + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + assert(self != NULL); + STAT_INC(CALL, hit); + volatile PyCFunctionFastWithKeywords cfunc_v = _PyCFunctionFastWithKeywords_CAST(cfunc); + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + callable, + cfunc_v, + self, + args, + oparg + ); + DEAD(args); + DEAD(callable); + ERROR_IF(res_o == NULL); + res = PyStackRef_FromPyObjectSteal(res_o); + } + macro(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) = _RECORD_CALLABLE + unused/1 + unused/2 + + _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + _CHECK_PERIODIC_AT_END; - op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res)) { - assert(oparg == 0 || oparg == 1); + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); + EXIT_IF(method->d_method->ml_flags != METH_NOARGS); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null)) { - args--; total_args++; } EXIT_IF(total_args != 1); + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); + EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); + } + + op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); - PyMethodDef *meth = method->d_method; + + assert(oparg == 1 || !PyStackRef_IsNull(self_or_null)); + if (!PyStackRef_IsNull(self_or_null)) { + args--; + } _PyStackRef self_stackref = args[0]; PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); - EXIT_IF(meth->ml_flags != METH_NOARGS); - // CPython promises to check all non-vectorcall function calls. - EXIT_IF(_Py_ReachedRecursionLimit(tstate)); STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; + PyCFunction cfunc = method->d_method->ml_meth; PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -4698,15 +4971,50 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } + tier2 op(_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE, (callable, args[oparg], cfunc/4 -- res)) { + assert(oparg == 1); + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); + STAT_INC(CALL, hit); + volatile PyCFunction cfunc_v = (PyCFunction)cfunc; + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc_v, self, NULL); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + DEAD(args); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL); + res = PyStackRef_FromPyObjectSteal(res_o); + } + macro(CALL_METHOD_DESCRIPTOR_NOARGS) = _RECORD_CALLABLE + unused/1 + unused/2 + + _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS + + _CHECK_RECURSION_LIMIT + _CALL_METHOD_DESCRIPTOR_NOARGS + _CHECK_PERIODIC_AT_END; + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + /* Builtin METH_FASTCALL methods, without keywords */ + EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); + EXIT_IF(method->d_method->ml_flags != METH_FASTCALL); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + EXIT_IF(total_args == 0); + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); + EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); + } + op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; int total_args = oparg; _PyStackRef *arguments = args; @@ -4714,19 +5022,13 @@ dummy_func( arguments--; total_args++; } - EXIT_IF(total_args == 0); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - /* Builtin METH_FASTCALL methods, without keywords */ - EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); - PyMethodDef *meth = method->d_method; - EXIT_IF(meth->ml_flags != METH_FASTCALL); PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); assert(self != NULL); - EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); STAT_INC(CALL, hit); + PyCFunctionFast cfunc = _PyCFunctionFast_CAST(method->d_method->ml_meth); PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( callable, - meth, + cfunc, self, arguments, total_args @@ -4738,9 +5040,28 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } + tier2 op(_CALL_METHOD_DESCRIPTOR_FAST_INLINE, (callable, args[oparg], cfunc/4 -- res)) { + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + assert(self != NULL); + STAT_INC(CALL, hit); + volatile PyCFunctionFast cfunc_v = _PyCFunctionFast_CAST(cfunc); + PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( + callable, + cfunc_v, + self, + args, + oparg + ); + DEAD(args); + DEAD(callable); + ERROR_IF(res_o == NULL); + res = PyStackRef_FromPyObjectSteal(res_o); + } + macro(CALL_METHOD_DESCRIPTOR_FAST) = unused/1 + unused/2 + + _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST + _CALL_METHOD_DESCRIPTOR_FAST + _CHECK_PERIODIC_AT_END; @@ -5857,6 +6178,17 @@ dummy_func( } } + tier2 op(_RECORD_3OS_GEN_FUNC, (gen, nos, tos -- gen, nos, tos)) { + PyObject *obj = PyStackRef_AsPyObjectBorrow(gen); + if (PyGen_Check(obj)) { + PyGenObject *gen_obj = (PyGenObject *)obj; + _PyStackRef func = gen_obj->gi_iframe.f_funcobj; + if (!PyStackRef_IsNull(func)) { + RECORD_VALUE(PyStackRef_AsPyObjectBorrow(func)); + } + } + } + tier2 op(_RECORD_4OS, (value, _3os, nos, tos -- value, _3os, nos, tos)) { RECORD_VALUE(PyStackRef_AsPyObjectBorrow(value)); } diff --git a/Python/ceval.c b/Python/ceval.c index 8a6895834cbb7e..4390e024dca5ef 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -602,7 +602,7 @@ _PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, if (allowed < nargs) { const char *plural = (allowed == 1) ? "" : "s"; _PyErr_Format(tstate, PyExc_TypeError, - "%s() accepts %d positional sub-pattern%s (%d given)", + "%s() accepts %zd positional sub-pattern%s (%zd given)", ((PyTypeObject*)type)->tp_name, allowed, plural, nargs); goto fail; @@ -874,7 +874,7 @@ _Py_BuiltinCallFastWithKeywords_StackRefSteal( PyObject * _PyCallMethodDescriptorFast_StackRefSteal( _PyStackRef callable, - PyMethodDef *meth, + PyCFunctionFast cfunc, PyObject *self, _PyStackRef *arguments, int total_args) @@ -885,10 +885,8 @@ _PyCallMethodDescriptorFast_StackRefSteal( res = NULL; goto cleanup; } - assert(((PyMethodDescrObject *)PyStackRef_AsPyObjectBorrow(callable))->d_method == meth); assert(self == PyStackRef_AsPyObjectBorrow(arguments[0])); - PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); res = cfunc(self, (args_o + 1), total_args - 1); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res != NULL) ^ (PyErr_Occurred() != NULL)); @@ -907,7 +905,7 @@ _PyCallMethodDescriptorFast_StackRefSteal( PyObject * _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( _PyStackRef callable, - PyMethodDef *meth, + PyCFunctionFastWithKeywords cfunc, PyObject *self, _PyStackRef *arguments, int total_args) @@ -918,11 +916,8 @@ _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( res = NULL; goto cleanup; } - assert(((PyMethodDescrObject *)PyStackRef_AsPyObjectBorrow(callable))->d_method == meth); assert(self == PyStackRef_AsPyObjectBorrow(arguments[0])); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); res = cfunc(self, (args_o + 1), total_args-1, NULL); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res != NULL) ^ (PyErr_Occurred() != NULL)); @@ -1165,6 +1160,55 @@ stop_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) #include "generated_cases.c.h" #endif + +_PyStackRef +_PyEval_GetIter(_PyStackRef iterable, _PyStackRef *index_or_null, int yield_from) +{ + PyTypeObject *tp = PyStackRef_TYPE(iterable); + if (tp == &PyTuple_Type || tp == &PyList_Type) { + /* Leave iterable on stack and pushed tagged 0 */ + *index_or_null = PyStackRef_TagInt(0); + return iterable; + } + *index_or_null = PyStackRef_NULL; + if (tp->tp_iter == PyObject_SelfIter) { + return iterable; + } + if (yield_from && tp == &PyCoro_Type) { + assert(yield_from != GET_ITER_YIELD_FROM); + if (yield_from == GET_ITER_YIELD_FROM_CORO_CHECK) { + /* `iterable` is a coroutine and it is used in a 'yield from' + * expression of a regular generator. */ + PyErr_SetString(PyExc_TypeError, + "cannot 'yield from' a coroutine object " + "in a non-coroutine generator"); + PyStackRef_CLOSE(iterable); + return PyStackRef_ERROR; + } + return iterable; + } + /* Pop iterable, and push iterator then NULL */ + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + PyStackRef_CLOSE(iterable); + if (iter_o == NULL) { + return PyStackRef_ERROR; + } + return PyStackRef_FromPyObjectSteal(iter_o); +} + +Py_NO_INLINE int +_Py_ReachedRecursionLimit(PyThreadState *tstate) { + uintptr_t here_addr = _Py_get_machine_stack_pointer(); + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + assert(_tstate->c_stack_hard_limit != 0); +#if _Py_STACK_GROWS_DOWN + return here_addr <= _tstate->c_stack_soft_limit; +#else + return here_addr >= _tstate->c_stack_soft_limit; +#endif +} + + #if (defined(__GNUC__) && __GNUC__ >= 10 && !defined(__clang__)) && defined(__x86_64__) /* * gh-129987: The SLP autovectorizer can cause poor code generation for @@ -1506,7 +1550,7 @@ format_missing(PyThreadState *tstate, const char *kind, if (name_str == NULL) return; _PyErr_Format(tstate, PyExc_TypeError, - "%U() missing %i required %s argument%s: %U", + "%U() missing %zd required %s argument%s: %U", qualname, len, kind, @@ -3050,7 +3094,7 @@ _PyEval_LazyImportName(PyThreadState *tstate, PyObject *builtins, break; } - if (!lazy) { + if (!lazy && PyImport_GetLazyImportsMode() != PyImport_LAZY_NONE) { // See if __lazy_modules__ forces this to be lazy. lazy = check_lazy_import_compatibility(tstate, globals, name, level); if (lazy < 0) { @@ -3407,40 +3451,36 @@ _Py_Check_ArgsIterable(PyThreadState *tstate, PyObject *func, PyObject *args) } void -_PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs) +_PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs, PyObject *dupkey) { - /* _PyDict_MergeEx raises attribute + if (dupkey != NULL) { + PyObject *funcstr = _PyObject_FunctionStr(func); + _PyErr_Format( + tstate, PyExc_TypeError, + "%V got multiple values for keyword argument '%S'", + funcstr, "finction", dupkey); + Py_XDECREF(funcstr); + return; + } + /* _PyDict_MergeUniq raises attribute * error (percolated from an attempt * to get 'keys' attribute) instead of * a type error if its second argument * is not a mapping. */ if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { - _PyErr_Format( - tstate, PyExc_TypeError, - "Value after ** must be a mapping, not %.200s", - Py_TYPE(kwargs)->tp_name); - } - else if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { PyObject *exc = _PyErr_GetRaisedException(tstate); - PyObject *args = PyException_GetArgs(exc); - if (PyTuple_Check(args) && PyTuple_GET_SIZE(args) == 1) { - _PyErr_Clear(tstate); - PyObject *funcstr = _PyObject_FunctionStr(func); - if (funcstr != NULL) { - PyObject *key = PyTuple_GET_ITEM(args, 0); - _PyErr_Format( - tstate, PyExc_TypeError, - "%U got multiple values for keyword argument '%S'", - funcstr, key); - Py_DECREF(funcstr); - } - Py_XDECREF(exc); + int has_keys = PyObject_HasAttrWithError(kwargs, &_Py_ID(keys)); + if (has_keys == 0) { + _PyErr_Format( + tstate, PyExc_TypeError, + "Value after ** must be a mapping, not %T", + kwargs); + Py_DECREF(exc); } else { - _PyErr_SetRaisedException(tstate, exc); + _PyErr_ChainExceptions1Tstate(tstate, exc); } - Py_DECREF(args); } } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index b127812b4bf703..ad790c038650f8 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -543,3 +543,65 @@ gen_try_set_executing(PyGenObject *gen) } return false; } + +// Macro for inplace float binary ops (tier 2 only). +// Mutates the uniquely-referenced TARGET operand in place. +// TARGET must be either left or right. +#define FLOAT_INPLACE_OP(left, right, TARGET, OP) \ + do { \ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); \ + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); \ + assert(PyFloat_CheckExact(left_o)); \ + assert(PyFloat_CheckExact(right_o)); \ + assert(_PyObject_IsUniquelyReferenced( \ + PyStackRef_AsPyObjectBorrow(TARGET))); \ + STAT_INC(BINARY_OP, hit); \ + double _dres = \ + ((PyFloatObject *)left_o)->ob_fval \ + OP ((PyFloatObject *)right_o)->ob_fval; \ + ((PyFloatObject *)PyStackRef_AsPyObjectBorrow(TARGET)) \ + ->ob_fval = _dres; \ + } while (0) + +// Inplace compact int operation. TARGET is expected to be uniquely +// referenced at the optimizer level, but at runtime it may be a +// cached small int singleton. We check _Py_IsImmortal on TARGET +// to decide whether inplace mutation is safe. +// +// After the macro, _int_inplace_res holds the result (may be NULL +// on allocation failure). On success, TARGET was mutated in place +// and _int_inplace_res is a DUP'd reference to it. On fallback +// (small int target, small int result, or overflow), _int_inplace_res +// is from FUNC (_PyCompactLong_Add etc.). +// FUNC is the fallback function (_PyCompactLong_Add etc.) +#define INT_INPLACE_OP(left, right, TARGET, OP, FUNC) \ + _PyStackRef _int_inplace_res = PyStackRef_NULL; \ + do { \ + PyObject *target_o = PyStackRef_AsPyObjectBorrow(TARGET); \ + if (_Py_IsImmortal(target_o)) { \ + break; \ + } \ + assert(_PyObject_IsUniquelyReferenced(target_o)); \ + Py_ssize_t left_val = _PyLong_CompactValue( \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(left)); \ + Py_ssize_t right_val = _PyLong_CompactValue( \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(right)); \ + Py_ssize_t result = left_val OP right_val; \ + if (!_PY_IS_SMALL_INT(result) \ + && ((twodigits)((stwodigits)result) + PyLong_MASK \ + < (twodigits)PyLong_MASK + PyLong_BASE)) \ + { \ + _PyLong_SetSignAndDigitCount( \ + (PyLongObject *)target_o, result < 0 ? -1 : 1, 1); \ + ((PyLongObject *)target_o)->long_value.ob_digit[0] = \ + (digit)(result < 0 ? -result : result); \ + _int_inplace_res = PyStackRef_DUP(TARGET); \ + break; \ + } \ + } while (0); \ + if (PyStackRef_IsNull(_int_inplace_res)) { \ + _int_inplace_res = FUNC( \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(left), \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(right)); \ + } + diff --git a/Python/codegen.c b/Python/codegen.c index 5749b615386717..aca590d055f466 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -606,6 +606,7 @@ codegen_unwind_fblock(compiler *c, location *ploc, RETURN_IF_ERROR(codegen_call_exit_with_nones(c, *ploc)); if (info->fb_type == COMPILE_FBLOCK_ASYNC_WITH) { ADDOP_I(c, *ploc, GET_AWAITABLE, 2); + ADDOP(c, *ploc, PUSH_NULL); ADDOP_LOAD_CONST(c, *ploc, Py_None); ADD_YIELD_FROM(c, *ploc, 1); } @@ -666,8 +667,8 @@ codegen_unwind_fblock_stack(compiler *c, location *ploc, _PyCompile_PopFBlock(c, top->fb_type, top->fb_block); RETURN_IF_ERROR(codegen_unwind_fblock(c, ploc, ©, preserve_tos)); RETURN_IF_ERROR(codegen_unwind_fblock_stack(c, ploc, preserve_tos, loop)); - _PyCompile_PushFBlock(c, copy.fb_loc, copy.fb_type, copy.fb_block, - copy.fb_exit, copy.fb_datum); + RETURN_IF_ERROR(_PyCompile_PushFBlock(c, copy.fb_loc, copy.fb_type, copy.fb_block, + copy.fb_exit, copy.fb_datum)); return SUCCESS; } @@ -714,10 +715,14 @@ codegen_setup_annotations_scope(compiler *c, location loc, // if .format > VALUE_WITH_FAKE_GLOBALS: raise NotImplementedError PyObject *value_with_fake_globals = PyLong_FromLong(_Py_ANNOTATE_FORMAT_VALUE_WITH_FAKE_GLOBALS); + if (value_with_fake_globals == NULL) { + return ERROR; + } + assert(!SYMTABLE_ENTRY(c)->ste_has_docstring); _Py_DECLARE_STR(format, ".format"); ADDOP_I(c, loc, LOAD_FAST, 0); - ADDOP_LOAD_CONST(c, loc, value_with_fake_globals); + ADDOP_LOAD_CONST_NEW(c, loc, value_with_fake_globals); ADDOP_I(c, loc, COMPARE_OP, (Py_GT << 5) | compare_masks[Py_GT]); NEW_JUMP_TARGET_LABEL(c, body); ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, body); @@ -793,6 +798,9 @@ codegen_deferred_annotations_body(compiler *c, location loc, if (!mangled) { return ERROR; } + // NOTE: ref of mangled can be leaked on ADDOP* and VISIT macros due to early returns + // fixing would require an overhaul of these macros + PyObject *cond_index = PyList_GET_ITEM(conditional_annotation_indices, i); assert(PyLong_CheckExact(cond_index)); long idx = PyLong_AS_LONG(cond_index); @@ -2124,7 +2132,7 @@ codegen_for(compiler *c, stmt_ty s) VISIT(c, expr, s->v.For.iter); loc = LOC(s->v.For.iter); - ADDOP(c, loc, GET_ITER); + ADDOP_I(c, loc, GET_ITER, 0); USE_LABEL(c, start); ADDOP_JUMP(c, loc, FOR_ITER, cleanup); @@ -2175,6 +2183,7 @@ codegen_async_for(compiler *c, stmt_ty s) /* SETUP_FINALLY to guard the __anext__ call */ ADDOP_JUMP(c, loc, SETUP_FINALLY, except); ADDOP(c, loc, GET_ANEXT); + ADDOP(c, loc, PUSH_NULL); ADDOP_LOAD_CONST(c, loc, Py_None); USE_LABEL(c, send); ADD_YIELD_FROM(c, loc, 1); @@ -3277,7 +3286,10 @@ codegen_nameop(compiler *c, location loc, } int scope = _PyST_GetScope(SYMTABLE_ENTRY(c), mangled); - RETURN_IF_ERROR(scope); + if (scope == -1) { + goto error; + } + _PyCompile_optype optype; Py_ssize_t arg = 0; if (_PyCompile_ResolveNameop(c, mangled, scope, &optype, &arg) < 0) { @@ -4540,7 +4552,7 @@ codegen_sync_comprehension_generator(compiler *c, location loc, if (IS_JUMP_TARGET_LABEL(start)) { if (iter_pos != ITERATOR_ON_STACK) { - ADDOP(c, LOC(gen->iter), GET_ITER); + ADDOP_I(c, LOC(gen->iter), GET_ITER, 0); depth += 1; } USE_LABEL(c, start); @@ -4574,7 +4586,7 @@ codegen_sync_comprehension_generator(compiler *c, location loc, NEW_JUMP_TARGET_LABEL(c, unpack_start); NEW_JUMP_TARGET_LABEL(c, unpack_end); VISIT(c, expr, elt->v.Starred.value); - ADDOP(c, elt_loc, GET_ITER); + ADDOP_I(c, elt_loc, GET_ITER, 0); USE_LABEL(c, unpack_start); ADDOP_JUMP(c, elt_loc, FOR_ITER, unpack_end); ADDOP_YIELD(c, elt_loc); @@ -4686,6 +4698,7 @@ codegen_async_comprehension_generator(compiler *c, location loc, ADDOP_JUMP(c, loc, SETUP_FINALLY, except); ADDOP(c, loc, GET_ANEXT); + ADDOP(c, loc, PUSH_NULL); ADDOP_LOAD_CONST(c, loc, Py_None); USE_LABEL(c, send); ADD_YIELD_FROM(c, loc, 1); @@ -4716,7 +4729,7 @@ codegen_async_comprehension_generator(compiler *c, location loc, NEW_JUMP_TARGET_LABEL(c, unpack_start); NEW_JUMP_TARGET_LABEL(c, unpack_end); VISIT(c, expr, elt->v.Starred.value); - ADDOP(c, elt_loc, GET_ITER); + ADDOP_I(c, elt_loc, GET_ITER, 0); USE_LABEL(c, unpack_start); ADDOP_JUMP(c, elt_loc, FOR_ITER, unpack_end); ADDOP_YIELD(c, elt_loc); @@ -5039,6 +5052,7 @@ codegen_comprehension(compiler *c, expr_ty e, int type, if (is_async_comprehension && type != COMP_GENEXP) { ADDOP_I(c, loc, GET_AWAITABLE, 0); + ADDOP(c, loc, PUSH_NULL); ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); } @@ -5178,6 +5192,7 @@ codegen_async_with_inner(compiler *c, stmt_ty s, int pos) ADDOP_I(c, loc, LOAD_SPECIAL, SPECIAL___AENTER__); ADDOP_I(c, loc, CALL, 0); ADDOP_I(c, loc, GET_AWAITABLE, 1); + ADDOP(c, loc, PUSH_NULL); ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); @@ -5214,6 +5229,7 @@ codegen_async_with_inner(compiler *c, stmt_ty s, int pos) */ RETURN_IF_ERROR(codegen_call_exit_with_nones(c, loc)); ADDOP_I(c, loc, GET_AWAITABLE, 2); + ADDOP(c, loc, PUSH_NULL); ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); @@ -5228,6 +5244,7 @@ codegen_async_with_inner(compiler *c, stmt_ty s, int pos) ADDOP(c, loc, PUSH_EXC_INFO); ADDOP(c, loc, WITH_EXCEPT_START); ADDOP_I(c, loc, GET_AWAITABLE, 2); + ADDOP(c, loc, PUSH_NULL); ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); RETURN_IF_ERROR(codegen_with_except_finish(c, cleanup)); @@ -5408,13 +5425,14 @@ codegen_visit_expr(compiler *c, expr_ty e) return _PyCompile_Error(c, loc, "'yield from' inside async function"); } VISIT(c, expr, e->v.YieldFrom.value); - ADDOP(c, loc, GET_YIELD_FROM_ITER); + ADDOP_I(c, loc, GET_ITER, GET_ITER_YIELD_FROM); ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 0); break; case Await_kind: VISIT(c, expr, e->v.Await.value); ADDOP_I(c, loc, GET_AWAITABLE, 0); + ADDOP(c, loc, PUSH_NULL); ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); break; diff --git a/Python/compile.c b/Python/compile.c index 4cf178b06ae11d..365b118cc71b44 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1100,18 +1100,22 @@ _PyCompile_TweakInlinedComprehensionScopes(compiler *c, location loc, assert(orig == NULL || orig == Py_True || orig == Py_False); if (orig != Py_True) { if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_True) < 0) { + Py_XDECREF(orig); return ERROR; } if (state->fast_hidden == NULL) { state->fast_hidden = PySet_New(NULL); if (state->fast_hidden == NULL) { + Py_XDECREF(orig); return ERROR; } } if (PySet_Add(state->fast_hidden, k) < 0) { + Py_XDECREF(orig); return ERROR; } } + Py_XDECREF(orig); } } } diff --git a/Python/crossinterp_data_lookup.h b/Python/crossinterp_data_lookup.h index c3c76ae8d9a289..54422ad2335cb6 100644 --- a/Python/crossinterp_data_lookup.h +++ b/Python/crossinterp_data_lookup.h @@ -455,7 +455,7 @@ _PyBytes_GetXIDataWrapped(PyThreadState *tstate, return NULL; } if (size < sizeof(_PyBytes_data_t)) { - PyErr_Format(PyExc_ValueError, "expected size >= %d, got %d", + PyErr_Format(PyExc_ValueError, "expected size >= %zu, got %zu", sizeof(_PyBytes_data_t), size); return NULL; } @@ -657,6 +657,7 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, xidata_fallback_t fallback, shared->items = (_PyXIData_t **) PyMem_Calloc(shared->len, sizeof(_PyXIData_t *)); if (shared->items == NULL) { PyErr_NoMemory(); + PyMem_RawFree(shared); return -1; } diff --git a/Python/dynload_shlib.c b/Python/dynload_shlib.c index 583c9b752dfd90..2e1455fbe232f4 100644 --- a/Python/dynload_shlib.c +++ b/Python/dynload_shlib.c @@ -44,7 +44,10 @@ const char *_PyImport_DynLoadFiletab[] = { #ifdef ALT_SOABI "." ALT_SOABI ".so", #endif +#ifndef Py_GIL_DISABLED ".abi" PYTHON_ABI_STRING ".so", +#endif /* Py_GIL_DISABLED */ + ".abi" PYTHON_ABI_STRING "t.so", ".so", #endif /* __CYGWIN__ */ NULL, diff --git a/Python/errors.c b/Python/errors.c index 229e3a565db5cf..48b03e5fd714b1 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -246,13 +246,23 @@ PyErr_SetObject(PyObject *exception, PyObject *value) _PyErr_SetObject(tstate, exception, value); } -/* Set a key error with the specified argument, wrapping it in a - * tuple automatically so that tuple keys are not unpacked as the - * exception arguments. */ +/* Set a key error with the specified argument. This function should be used to + * raise a KeyError with an argument instead of PyErr_SetObject(PyExc_KeyError, + * arg) which has a special behavior. PyErr_SetObject() unpacks arg if it's a + * tuple, and it uses arg instead of creating a new exception if arg is an + * exception. + * + * If an exception is already set, override the exception. */ void _PyErr_SetKeyError(PyObject *arg) { PyThreadState *tstate = _PyThreadState_GET(); + + // PyObject_CallOneArg() must not be called with an exception set, + // otherwise _Py_CheckFunctionResult() can fail if the function returned + // a result with an excception set. + _PyErr_Clear(tstate); + PyObject *exc = PyObject_CallOneArg(PyExc_KeyError, arg); if (!exc) { /* caller will expect error to be set anyway */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index fbbfd521e0d74c..6600accb37e3f2 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2602,17 +2602,21 @@ break; } - case _END_SEND_r21: { - CHECK_CURRENT_CACHED_VALUES(2); + case _END_SEND_r31: { + CHECK_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef value; + _PyStackRef index_or_null; _PyStackRef receiver; _PyStackRef val; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + index_or_null = _stack_item_1; receiver = _stack_item_0; val = value; + (void)index_or_null; stack_pointer[0] = val; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2659,6 +2663,78 @@ break; } + case _UNARY_NEGATIVE_FLOAT_INPLACE_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + value = stack_pointer[-1]; + PyObject *val_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyFloat_CheckExact(val_o)); + assert(_PyObject_IsUniquelyReferenced(val_o)); + STAT_INC(UNARY_NEGATIVE, hit); + double dres = -((PyFloatObject *)val_o)->ob_fval; + ((PyFloatObject *)val_o)->ob_fval = dres; + res = value; + v = PyStackRef_NULL; + _tos_cache1 = v; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNARY_NEGATIVE_FLOAT_INPLACE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *val_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyFloat_CheckExact(val_o)); + assert(_PyObject_IsUniquelyReferenced(val_o)); + STAT_INC(UNARY_NEGATIVE, hit); + double dres = -((PyFloatObject *)val_o)->ob_fval; + ((PyFloatObject *)val_o)->ob_fval = dres; + res = value; + v = PyStackRef_NULL; + _tos_cache1 = v; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNARY_NEGATIVE_FLOAT_INPLACE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + PyObject *val_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyFloat_CheckExact(val_o)); + assert(_PyObject_IsUniquelyReferenced(val_o)); + STAT_INC(UNARY_NEGATIVE, hit); + double dres = -((PyFloatObject *)val_o)->ob_fval; + ((PyFloatObject *)val_o)->ob_fval = dres; + res = value; + v = PyStackRef_NULL; + _tos_cache2 = v; + _tos_cache1 = res; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + case _UNARY_NOT_r01: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -4428,181 +4504,189 @@ break; } - case _GUARD_NOS_FLOAT_r02: { + case _BINARY_OP_ADD_INT_INPLACE_r03: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { + INT_INPLACE_OP(left, right, left, +, _PyCompactLong_Add); + if (PyStackRef_IsNull(_int_inplace_res)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _GUARD_NOS_FLOAT_r12: { + case _BINARY_OP_ADD_INT_INPLACE_r13: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { + INT_INPLACE_OP(left, right, left, +, _PyCompactLong_Add); + if (PyStackRef_IsNull(_int_inplace_res)) { UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; + _tos_cache0 = right; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } - _tos_cache1 = _stack_item_0; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _GUARD_NOS_FLOAT_r22: { + case _BINARY_OP_ADD_INT_INPLACE_r23: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { + INT_INPLACE_OP(left, right, left, +, _PyCompactLong_Add); + if (PyStackRef_IsNull(_int_inplace_res)) { UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; + _tos_cache1 = right; _tos_cache0 = left; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - _tos_cache1 = _stack_item_1; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_FLOAT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - left = _stack_item_1; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _GUARD_TOS_FLOAT_r01: { + case _BINARY_OP_SUBTRACT_INT_INPLACE_r03: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + INT_INPLACE_OP(left, right, left, -, _PyCompactLong_Subtract); + if (PyStackRef_IsNull(_int_inplace_res)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _GUARD_TOS_FLOAT_r11: { + case _BINARY_OP_SUBTRACT_INT_INPLACE_r13: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { + right = _stack_item_0; + left = stack_pointer[-1]; + INT_INPLACE_OP(left, right, left, -, _PyCompactLong_Subtract); + if (PyStackRef_IsNull(_int_inplace_res)) { UOP_STAT_INC(uopcode, miss); - _tos_cache0 = value; + _tos_cache0 = right; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _GUARD_TOS_FLOAT_r22: { + case _BINARY_OP_SUBTRACT_INT_INPLACE_r23: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { + right = _stack_item_1; + left = _stack_item_0; + INT_INPLACE_OP(left, right, left, -, _PyCompactLong_Subtract); + if (PyStackRef_IsNull(_int_inplace_res)) { UOP_STAT_INC(uopcode, miss); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; + _tos_cache1 = right; + _tos_cache0 = left; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_FLOAT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - value = _stack_item_2; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _BINARY_OP_MULTIPLY_FLOAT_r03: { + case _BINARY_OP_MULTIPLY_INT_INPLACE_r03: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; @@ -4612,20 +4696,13 @@ _PyStackRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval * - ((PyFloatObject *)right_o)->ob_fval; - PyObject *d = PyFloat_FromDouble(dres); - if (d == NULL) { + INT_INPLACE_OP(left, right, left, *, _PyCompactLong_Multiply); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + JUMP_TO_JUMP_TARGET(); } - res = PyStackRef_FromPyObjectSteal(d); + res = _int_inplace_res; l = left; r = right; _tos_cache2 = r; @@ -4638,7 +4715,7 @@ break; } - case _BINARY_OP_MULTIPLY_FLOAT_r13: { + case _BINARY_OP_MULTIPLY_INT_INPLACE_r13: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; @@ -4649,23 +4726,14 @@ _PyStackRef _stack_item_0 = _tos_cache0; right = _stack_item_0; left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval * - ((PyFloatObject *)right_o)->ob_fval; - PyObject *d = PyFloat_FromDouble(dres); - if (d == NULL) { - stack_pointer[0] = right; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + INT_INPLACE_OP(left, right, left, *, _PyCompactLong_Multiply); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } - res = PyStackRef_FromPyObjectSteal(d); + res = _int_inplace_res; l = left; r = right; _tos_cache2 = r; @@ -4678,7 +4746,7 @@ break; } - case _BINARY_OP_MULTIPLY_FLOAT_r23: { + case _BINARY_OP_MULTIPLY_INT_INPLACE_r23: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; @@ -4690,24 +4758,15 @@ _PyStackRef _stack_item_1 = _tos_cache1; right = _stack_item_1; left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval * - ((PyFloatObject *)right_o)->ob_fval; - PyObject *d = PyFloat_FromDouble(dres); - if (d == NULL) { - stack_pointer[0] = left; - stack_pointer[1] = right; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + INT_INPLACE_OP(left, right, left, *, _PyCompactLong_Multiply); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - res = PyStackRef_FromPyObjectSteal(d); + res = _int_inplace_res; l = left; r = right; _tos_cache2 = r; @@ -4718,7 +4777,7 @@ break; } - case _BINARY_OP_ADD_FLOAT_r03: { + case _BINARY_OP_ADD_INT_INPLACE_RIGHT_r03: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; @@ -4728,20 +4787,13 @@ _PyStackRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval + - ((PyFloatObject *)right_o)->ob_fval; - PyObject *d = PyFloat_FromDouble(dres); - if (d == NULL) { + INT_INPLACE_OP(left, right, right, +, _PyCompactLong_Add); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + JUMP_TO_JUMP_TARGET(); } - res = PyStackRef_FromPyObjectSteal(d); + res = _int_inplace_res; l = left; r = right; _tos_cache2 = r; @@ -4754,7 +4806,7 @@ break; } - case _BINARY_OP_ADD_FLOAT_r13: { + case _BINARY_OP_ADD_INT_INPLACE_RIGHT_r13: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; @@ -4765,23 +4817,14 @@ _PyStackRef _stack_item_0 = _tos_cache0; right = _stack_item_0; left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval + - ((PyFloatObject *)right_o)->ob_fval; - PyObject *d = PyFloat_FromDouble(dres); - if (d == NULL) { - stack_pointer[0] = right; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + INT_INPLACE_OP(left, right, right, +, _PyCompactLong_Add); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } - res = PyStackRef_FromPyObjectSteal(d); + res = _int_inplace_res; l = left; r = right; _tos_cache2 = r; @@ -4794,7 +4837,7 @@ break; } - case _BINARY_OP_ADD_FLOAT_r23: { + case _BINARY_OP_ADD_INT_INPLACE_RIGHT_r23: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; @@ -4806,24 +4849,15 @@ _PyStackRef _stack_item_1 = _tos_cache1; right = _stack_item_1; left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval + - ((PyFloatObject *)right_o)->ob_fval; - PyObject *d = PyFloat_FromDouble(dres); - if (d == NULL) { - stack_pointer[0] = left; - stack_pointer[1] = right; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + INT_INPLACE_OP(left, right, right, +, _PyCompactLong_Add); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - res = PyStackRef_FromPyObjectSteal(d); + res = _int_inplace_res; l = left; r = right; _tos_cache2 = r; @@ -4834,7 +4868,7 @@ break; } - case _BINARY_OP_SUBTRACT_FLOAT_r03: { + case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; @@ -4844,20 +4878,13 @@ _PyStackRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval - - ((PyFloatObject *)right_o)->ob_fval; - PyObject *d = PyFloat_FromDouble(dres); - if (d == NULL) { + INT_INPLACE_OP(left, right, right, -, _PyCompactLong_Subtract); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + JUMP_TO_JUMP_TARGET(); } - res = PyStackRef_FromPyObjectSteal(d); + res = _int_inplace_res; l = left; r = right; _tos_cache2 = r; @@ -4870,7 +4897,7 @@ break; } - case _BINARY_OP_SUBTRACT_FLOAT_r13: { + case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; @@ -4881,23 +4908,14 @@ _PyStackRef _stack_item_0 = _tos_cache0; right = _stack_item_0; left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval - - ((PyFloatObject *)right_o)->ob_fval; - PyObject *d = PyFloat_FromDouble(dres); - if (d == NULL) { - stack_pointer[0] = right; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + INT_INPLACE_OP(left, right, right, -, _PyCompactLong_Subtract); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } - res = PyStackRef_FromPyObjectSteal(d); + res = _int_inplace_res; l = left; r = right; _tos_cache2 = r; @@ -4910,7 +4928,7 @@ break; } - case _BINARY_OP_SUBTRACT_FLOAT_r23: { + case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; @@ -4922,24 +4940,15 @@ _PyStackRef _stack_item_1 = _tos_cache1; right = _stack_item_1; left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval - - ((PyFloatObject *)right_o)->ob_fval; - PyObject *d = PyFloat_FromDouble(dres); - if (d == NULL) { - stack_pointer[0] = left; - stack_pointer[1] = right; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + INT_INPLACE_OP(left, right, right, -, _PyCompactLong_Subtract); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - res = PyStackRef_FromPyObjectSteal(d); + res = _int_inplace_res; l = left; r = right; _tos_cache2 = r; @@ -4950,7 +4959,7 @@ break; } - case _BINARY_OP_ADD_UNICODE_r03: { + case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; @@ -4960,17 +4969,13 @@ _PyStackRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyUnicode_Concat(left_o, right_o); - if (res_o == NULL) { + INT_INPLACE_OP(left, right, right, *, _PyCompactLong_Multiply); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + JUMP_TO_JUMP_TARGET(); } - res = PyStackRef_FromPyObjectSteal(res_o); + res = _int_inplace_res; l = left; r = right; _tos_cache2 = r; @@ -4983,7 +4988,7 @@ break; } - case _BINARY_OP_ADD_UNICODE_r13: { + case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; @@ -4994,20 +4999,14 @@ _PyStackRef _stack_item_0 = _tos_cache0; right = _stack_item_0; left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyUnicode_Concat(left_o, right_o); - if (res_o == NULL) { - stack_pointer[0] = right; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + INT_INPLACE_OP(left, right, right, *, _PyCompactLong_Multiply); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } - res = PyStackRef_FromPyObjectSteal(res_o); + res = _int_inplace_res; l = left; r = right; _tos_cache2 = r; @@ -5020,7 +5019,7 @@ break; } - case _BINARY_OP_ADD_UNICODE_r23: { + case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; @@ -5032,21 +5031,15 @@ _PyStackRef _stack_item_1 = _tos_cache1; right = _stack_item_1; left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyUnicode_Concat(left_o, right_o); - if (res_o == NULL) { - stack_pointer[0] = left; - stack_pointer[1] = right; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + INT_INPLACE_OP(left, right, right, *, _PyCompactLong_Multiply); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - res = PyStackRef_FromPyObjectSteal(res_o); + res = _int_inplace_res; l = left; r = right; _tos_cache2 = r; @@ -5057,62 +5050,1129 @@ break; } - case _BINARY_OP_INPLACE_ADD_UNICODE_r21: { - CHECK_CURRENT_CACHED_VALUES(2); + case _GUARD_NOS_FLOAT_r02: { + CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; _PyStackRef left; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right))); - int next_oparg; - #if TIER_ONE - assert(next_instr->op.code == STORE_FAST); - next_oparg = next_instr->op.arg; - #else - next_oparg = (int)CURRENT_OPERAND0_16(); - #endif - _PyStackRef *target_local = &GETLOCAL(next_oparg); - assert(PyUnicode_CheckExact(left_o)); - if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { + if (!PyFloat_CheckExact(left_o)) { UOP_STAT_INC(uopcode, miss); - _tos_cache1 = right; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(BINARY_OP, hit); - assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left)); - PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); - PyObject *right_o = PyStackRef_AsPyObjectSteal(right); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyUnicode_Append(&temp, right_o); - _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); - stack_pointer = _PyFrame_GetStackPointer(frame); - *target_local = PyStackRef_NULL; - if (temp == NULL) { SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + JUMP_TO_JUMP_TARGET(); } - res = PyStackRef_FromPyObjectSteal(temp); - _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _GUARD_BINARY_OP_EXTEND_r22: { - CHECK_CURRENT_CACHED_VALUES(2); + case _GUARD_NOS_FLOAT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_0; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_FLOAT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_FLOAT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + left = _stack_item_1; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_FLOAT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_FLOAT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_FLOAT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_FLOAT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + stack_pointer[0] = right; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_FLOAT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_FLOAT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + stack_pointer[0] = right; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_FLOAT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + stack_pointer[0] = right; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_FLOAT_INPLACE_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + FLOAT_INPLACE_OP(left, right, left, +); + res = left; + l = PyStackRef_NULL; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_FLOAT_INPLACE_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + FLOAT_INPLACE_OP(left, right, left, +); + res = left; + l = PyStackRef_NULL; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_FLOAT_INPLACE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + FLOAT_INPLACE_OP(left, right, left, +); + res = left; + l = PyStackRef_NULL; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + FLOAT_INPLACE_OP(left, right, left, -); + res = left; + l = PyStackRef_NULL; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + FLOAT_INPLACE_OP(left, right, left, -); + res = left; + l = PyStackRef_NULL; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + FLOAT_INPLACE_OP(left, right, left, -); + res = left; + l = PyStackRef_NULL; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + FLOAT_INPLACE_OP(left, right, left, *); + res = left; + l = PyStackRef_NULL; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + FLOAT_INPLACE_OP(left, right, left, *); + res = left; + l = PyStackRef_NULL; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + FLOAT_INPLACE_OP(left, right, left, *); + res = left; + l = PyStackRef_NULL; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + FLOAT_INPLACE_OP(left, right, right, +); + res = right; + l = left; + r = PyStackRef_NULL; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + FLOAT_INPLACE_OP(left, right, right, +); + res = right; + l = left; + r = PyStackRef_NULL; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + FLOAT_INPLACE_OP(left, right, right, +); + res = right; + l = left; + r = PyStackRef_NULL; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + FLOAT_INPLACE_OP(left, right, right, *); + res = right; + l = left; + r = PyStackRef_NULL; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + FLOAT_INPLACE_OP(left, right, right, *); + res = right; + l = left; + r = PyStackRef_NULL; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + FLOAT_INPLACE_OP(left, right, right, *); + res = right; + l = left; + r = PyStackRef_NULL; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + FLOAT_INPLACE_OP(left, right, right, -); + res = right; + l = left; + r = PyStackRef_NULL; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + FLOAT_INPLACE_OP(left, right, right, -); + res = right; + l = left; + r = PyStackRef_NULL; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + FLOAT_INPLACE_OP(left, right, right, -); + res = right; + l = left; + r = PyStackRef_NULL; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_UNICODE_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_UNICODE_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + if (res_o == NULL) { + stack_pointer[0] = right; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_UNICODE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + if (res_o == NULL) { + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_INPLACE_ADD_UNICODE_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right))); + int next_oparg; + #if TIER_ONE + assert(next_instr->op.code == STORE_FAST); + next_oparg = next_instr->op.arg; + #else + next_oparg = (int)CURRENT_OPERAND0_16(); + #endif + _PyStackRef *target_local = &GETLOCAL(next_oparg); + assert(PyUnicode_CheckExact(left_o)); + if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(BINARY_OP, hit); + assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left)); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); + PyObject *right_o = PyStackRef_AsPyObjectSteal(right); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyUnicode_Append(&temp, right_o); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + stack_pointer = _PyFrame_GetStackPointer(frame); + *target_local = PyStackRef_NULL; + if (temp == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(temp); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_BINARY_OP_EXTEND_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; _PyStackRef left; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -6392,6 +7452,53 @@ break; } + case _BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef dict_st; + _PyStackRef res; + _PyStackRef ds; + _PyStackRef ss; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + sub_st = _stack_item_1; + dict_st = _stack_item_0; + PyObject *hash = (PyObject *)CURRENT_OPERAND0_64(); + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyAnyDict_CheckExact(dict)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o; + stack_pointer[0] = dict_st; + stack_pointer[1] = sub_st; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int rc = _PyDict_GetItemRef_KnownHash((PyDictObject *)dict, sub, (Py_hash_t)hash, &res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (rc == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetKeyError(sub); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + if (rc <= 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + ds = dict_st; + ss = sub_st; + _tos_cache2 = ss; + _tos_cache1 = ds; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + case _BINARY_OP_SUBSCR_DICT_r23: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -6755,7 +7862,53 @@ break; } - case _STORE_SUBSCR_DICT_r31: { + case _STORE_SUBSCR_DICT_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub; + _PyStackRef dict_st; + _PyStackRef value; + _PyStackRef st; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + sub = _stack_item_2; + dict_st = _stack_item_1; + value = _stack_item_0; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + STAT_INC(STORE_SUBSCR, hit); + stack_pointer[0] = value; + stack_pointer[1] = dict_st; + stack_pointer[2] = sub; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + st = dict_st; + _tos_cache0 = st; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _STORE_SUBSCR_DICT_KNOWN_HASH_r31: { CHECK_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef sub; @@ -6768,6 +7921,7 @@ sub = _stack_item_2; dict_st = _stack_item_1; value = _stack_item_0; + PyObject *hash = (PyObject *)CURRENT_OPERAND0_64(); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); assert(PyDict_CheckExact(dict)); STAT_INC(STORE_SUBSCR, hit); @@ -6777,9 +7931,10 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + int err = _PyDict_SetItem_Take2_KnownHash((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(sub), - PyStackRef_AsPyObjectSteal(value)); + PyStackRef_AsPyObjectSteal(value), + (Py_hash_t)hash); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { stack_pointer += -3; @@ -6872,12 +8027,14 @@ break; } - case _CALL_INTRINSIC_2_r21: { + case _CALL_INTRINSIC_2_r23: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef value1_st; _PyStackRef value2_st; _PyStackRef res; + _PyStackRef vs1; + _PyStackRef vs2; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); @@ -6892,26 +8049,20 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - _PyStackRef tmp = value1_st; - value1_st = PyStackRef_NULL; - stack_pointer[-1] = value1_st; - PyStackRef_CLOSE(tmp); - tmp = value2_st; - value2_st = PyStackRef_NULL; - stack_pointer[-2] = value2_st; - PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); + vs1 = value1_st; + vs2 = value2_st; + _tos_cache2 = vs2; + _tos_cache1 = vs1; _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } @@ -7132,30 +8283,33 @@ /* _SEND is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _SEND_GEN_FRAME_r22: { - CHECK_CURRENT_CACHED_VALUES(2); + case _SEND_GEN_FRAME_r33: { + CHECK_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef v; _PyStackRef receiver; _PyStackRef gen_frame; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; oparg = CURRENT_OPARG(); - v = _stack_item_1; + v = _stack_item_2; receiver = _stack_item_0; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { UOP_STAT_INC(uopcode, miss); - _tos_cache1 = v; + _tos_cache2 = v; + _tos_cache1 = _stack_item_1; _tos_cache0 = receiver; - SET_CURRENT_CACHED_VALUES(2); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } if (!gen_try_set_executing((PyGenObject *)gen)) { UOP_STAT_INC(uopcode, miss); - _tos_cache1 = v; + _tos_cache2 = v; + _tos_cache1 = _stack_item_1; _tos_cache0 = receiver; - SET_CURRENT_CACHED_VALUES(2); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } STAT_INC(SEND, hit); @@ -7167,10 +8321,10 @@ frame->return_offset = (uint16_t)( 2u + oparg); pushed_frame->previous = frame; gen_frame = PyStackRef_Wrap(pushed_frame); - _tos_cache1 = gen_frame; + _tos_cache2 = gen_frame; + _tos_cache1 = _stack_item_1; _tos_cache0 = receiver; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(2); + SET_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } @@ -7478,6 +8632,121 @@ break; } + case _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + seq = stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + _tos_cache1 = val0; + _tos_cache0 = val1; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + _PyStackRef _stack_item_0 = _tos_cache0; + seq = _stack_item_0; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + _tos_cache1 = val0; + _tos_cache0 = val1; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + seq = _stack_item_1; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + _tos_cache2 = val0; + _tos_cache1 = val1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef seq; + _PyStackRef val2; + _PyStackRef val1; + _PyStackRef val0; + seq = stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); + val2 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 2)); + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + _tos_cache2 = val0; + _tos_cache1 = val1; + _tos_cache0 = val2; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef seq; + _PyStackRef val2; + _PyStackRef val1; + _PyStackRef val0; + _PyStackRef _stack_item_0 = _tos_cache0; + seq = _stack_item_0; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); + val2 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 2)); + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + _tos_cache2 = val0; + _tos_cache1 = val1; + _tos_cache0 = val2; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + case _UNPACK_SEQUENCE_TUPLE_r10: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -7513,6 +8782,33 @@ break; } + case _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef seq; + _PyStackRef *values; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + seq = _stack_item_0; + values = &stack_pointer[0]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + assert(PyTuple_CheckExact(seq_o)); + assert(PyTuple_GET_SIZE(seq_o) == oparg); + assert(_PyObject_IsUniquelyReferenced(seq_o)); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectSteal(items[i]); + } + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + case _UNPACK_SEQUENCE_LIST_r10: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -8518,11 +9814,12 @@ break; } - case _LIST_EXTEND_r10: { + case _LIST_EXTEND_r11: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef iterable_st; _PyStackRef list_st; + _PyStackRef i; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); iterable_st = _stack_item_0; @@ -8549,24 +9846,17 @@ Py_TYPE(iterable)->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); } - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable_st); - stack_pointer = _PyFrame_GetStackPointer(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } assert(Py_IsNone(none_val)); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; + i = iterable_st; + _tos_cache0 = i; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } @@ -8739,11 +10029,12 @@ break; } - case _DICT_UPDATE_r10: { + case _DICT_UPDATE_r11: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef update; _PyStackRef dict; + _PyStackRef upd; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); update = _stack_item_0; @@ -8762,38 +10053,44 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (matches) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update_o)->tp_name); + PyObject *exc = _PyErr_GetRaisedException(tstate); + int has_keys = PyObject_HasAttrWithError(update_o, &_Py_ID(keys)); stack_pointer = _PyFrame_GetStackPointer(frame); + if (has_keys == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'%T' object is not a mapping", + update_o); + Py_DECREF(exc); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_ChainExceptions1(exc); + stack_pointer = _PyFrame_GetStackPointer(frame); + } } - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; + upd = update; + _tos_cache0 = upd; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _DICT_MERGE_r10: { + case _DICT_MERGE_r11: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef update; _PyStackRef dict; _PyStackRef callable; + _PyStackRef u; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); update = _stack_item_0; @@ -8802,33 +10099,28 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + PyObject *dupkey = NULL; stack_pointer[0] = update; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_MergeEx(dict_o, update_o, 2); + int err = _PyDict_MergeUniq(dict_o, update_o, &dupkey); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatKwargsError(tstate, callable_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); + _PyEval_FormatKwargsError(tstate, callable_o, update_o, dupkey); + Py_XDECREF(dupkey); stack_pointer = _PyFrame_GetStackPointer(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; + u = update; + _tos_cache0 = u; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } @@ -11493,98 +12785,30 @@ _PyStackRef iter; _PyStackRef index_or_null; _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); iterable = _stack_item_0; #ifdef Py_STATS + _Py_GatherStats_GetIter(iterable); + #endif stack_pointer[0] = iterable; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_GatherStats_GetIter(iterable); + _PyStackRef result = _PyEval_GetIter(iterable, &index_or_null, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - #endif - PyTypeObject *tp = PyStackRef_TYPE(iterable); - if (tp == &PyTuple_Type || tp == &PyList_Type) { - iter = iterable; - index_or_null = PyStackRef_TagInt(0); - } - else { - stack_pointer[0] = iterable; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsError(result)) { stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - index_or_null = PyStackRef_NULL; + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); } + iter = result; _tos_cache1 = index_or_null; _tos_cache0 = iter; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GET_YIELD_FROM_ITER_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iterable; - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - iterable = _stack_item_0; - PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); - if (PyCoro_CheckExact(iterable_o)) { - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - stack_pointer[0] = iterable; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - iter = iterable; - } - else if (PyGen_CheckExact(iterable_o)) { - iter = iterable; - } - else { - stack_pointer[0] = iterable; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(iterable_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = iterable; - iterable = iter; - stack_pointer[-1] = iterable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - } - _tos_cache0 = iter; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } @@ -15020,40 +16244,56 @@ break; } - case _CALL_BUILTIN_O_r03: { + case _GUARD_CALLABLE_BUILTIN_O_r00: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; - _PyStackRef res; - _PyStackRef c; - _PyStackRef s; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - if (total_args != 1) { + if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (!PyCFunction_CheckExact(callable_o)) { + if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args != 1) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_BUILTIN_O_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + _PyStackRef c; + _PyStackRef s; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyStackRef_IsNull(self_or_null)) { + args--; + } if (_Py_ReachedRecursionLimit(tstate)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); @@ -15084,6 +16324,28 @@ break; } + case _GUARD_CALLABLE_BUILTIN_FAST_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyCFunction_CheckExact(callable_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + case _CALL_BUILTIN_FAST_r01: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -15101,17 +16363,6 @@ arguments--; total_args++; } - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyCFunction_CheckExact(callable_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _Py_BuiltinCallFast_StackRefSteal( @@ -15137,6 +16388,28 @@ break; } + case _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyCFunction_CheckExact(callable_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + case _CALL_BUILTIN_FAST_WITH_KEYWORDS_r01: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -15154,17 +16427,6 @@ arguments--; total_args++; } - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyCFunction_CheckExact(callable_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRefSteal(callable, arguments, total_args); @@ -15761,81 +17023,249 @@ break; } + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (method->d_method->ml_flags != METH_O) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args != 2) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + case _CALL_METHOD_DESCRIPTOR_O_r03: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; - _PyStackRef res; - _PyStackRef c; - _PyStackRef s; - _PyStackRef a; + _PyStackRef res; + _PyStackRef c; + _PyStackRef s; + _PyStackRef a; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + } + STAT_INC(CALL, hit); + PyCFunction cfunc = method->d_method->ml_meth; + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + PyObject *arg = PyStackRef_AsPyObjectBorrow(arguments[1]); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + c = callable; + s = arguments[0]; + a = arguments[1]; + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache2 = a; + _tos_cache1 = s; + _tos_cache0 = c; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_RECURSION_LIMIT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + if (_Py_ReachedRecursionLimit(tstate)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_RECURSION_LIMIT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + if (_Py_ReachedRecursionLimit(tstate)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_RECURSION_LIMIT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + if (_Py_ReachedRecursionLimit(tstate)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_RECURSION_LIMIT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + if (_Py_ReachedRecursionLimit(tstate)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_O_INLINE_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef callable; + _PyStackRef res; + _PyStackRef c; + _PyStackRef s; + _PyStackRef a; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + callable = stack_pointer[-1 - oparg]; + PyObject *cfunc = (PyObject *)CURRENT_OPERAND0_64(); + assert(oparg == 2); + STAT_INC(CALL, hit); + volatile PyCFunction cfunc_v = (PyCFunction)cfunc; + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *arg = PyStackRef_AsPyObjectBorrow(args[1]); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc_v, self, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + c = callable; + s = args[0]; + a = args[1]; + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache2 = a; + _tos_cache1 = s; + _tos_cache0 = c; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer[-1 - oparg] = res; + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (total_args != 2) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_O) { + if (method->d_method->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (_Py_ReachedRecursionLimit(tstate)) { + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (total_args == 0) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - _PyStackRef arg_stackref = arguments[1]; - _PyStackRef self_stackref = arguments[0]; - if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type)) { + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, - PyStackRef_AsPyObjectBorrow(self_stackref), - PyStackRef_AsPyObjectBorrow(arg_stackref)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - c = callable; - s = arguments[0]; - a = arguments[1]; - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache2 = a; - _tos_cache1 = s; - _tos_cache0 = c; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } @@ -15852,42 +17282,21 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } - if (total_args == 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); assert(self != NULL); - if (!Py_IS_TYPE(self, d_type)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords_CAST(method->d_method->ml_meth); PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( callable, - meth, + cfunc, self, arguments, total_args @@ -15910,55 +17319,110 @@ break; } - case _CALL_METHOD_DESCRIPTOR_NOARGS_r01: { + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r01: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef *args; - _PyStackRef self_or_null; _PyStackRef callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; + callable = stack_pointer[-1 - oparg]; + PyObject *cfunc = (PyObject *)CURRENT_OPERAND0_64(); + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + assert(self != NULL); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + volatile PyCFunctionFastWithKeywords cfunc_v = _PyCFunctionFastWithKeywords_CAST(cfunc); + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + callable, + cfunc_v, + self, + args, + oparg + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - if (total_args != 1) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - PyMethodDef *meth = method->d_method; - _PyStackRef self_stackref = args[0]; - PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - if (!Py_IS_TYPE(self, method->d_common.d_type)) { + if (method->d_method->ml_flags != METH_NOARGS) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (meth->ml_flags != METH_NOARGS) { + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args != 1) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (_Py_ReachedRecursionLimit(tstate)) { + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_NOARGS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + assert(oparg == 1 || !PyStackRef_IsNull(self_or_null)); + if (!PyStackRef_IsNull(self_or_null)) { + args--; + } + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; + PyCFunction cfunc = method->d_method->ml_meth; _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -15985,53 +17449,117 @@ break; } - case _CALL_METHOD_DESCRIPTOR_FAST_r01: { + case _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r01: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef *args; - _PyStackRef self_or_null; _PyStackRef callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; + callable = stack_pointer[-1 - oparg]; + PyObject *cfunc = (PyObject *)CURRENT_OPERAND0_64(); + assert(oparg == 1); + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); + STAT_INC(CALL, hit); + volatile PyCFunction cfunc_v = (PyCFunction)cfunc; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc_v, self, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self_stackref); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (total_args == 0) { + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + if (method->d_method->ml_flags != METH_FASTCALL) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_FASTCALL) { + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args == 0) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - assert(self != NULL); + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); if (!Py_IS_TYPE(self, method->d_common.d_type)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_FAST_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFast cfunc = _PyCFunctionFast_CAST(method->d_method->ml_meth); PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( callable, - meth, + cfunc, self, arguments, total_args @@ -16054,6 +17582,46 @@ break; } + case _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + callable = stack_pointer[-1 - oparg]; + PyObject *cfunc = (PyObject *)CURRENT_OPERAND0_64(); + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + assert(self != NULL); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + volatile PyCFunctionFast cfunc_v = _PyCFunctionFast_CAST(cfunc); + PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( + callable, + cfunc_v, + self, + args, + oparg + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + /* _MONITOR_CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _MAYBE_EXPAND_METHOD_KW_r11: { diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 04234a6025468e..e988f4451007fb 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -1411,7 +1411,7 @@ maybe_instr_make_load_smallint(cfg_instr *instr, PyObject *newconst, if (val == -1 && PyErr_Occurred()) { return -1; } - if (!overflow && _PY_IS_SMALL_INT(val)) { + if (!overflow && _PY_IS_SMALL_INT(val) && 0 <= val && val <= 255) { assert(_Py_IsImmortal(newconst)); INSTR_SET_OP1(instr, LOAD_SMALL_INT, (int)val); return 1; @@ -2884,7 +2884,6 @@ optimize_load_fast(cfg_builder *g) case GET_ANEXT: case GET_ITER: case GET_LEN: - case GET_YIELD_FROM_ITER: case IMPORT_FROM: case MATCH_KEYS: case MATCH_MAPPING: @@ -2919,7 +2918,16 @@ optimize_load_fast(cfg_builder *g) break; } - case END_SEND: + case END_SEND: { + assert(_PyOpcode_num_popped(opcode, oparg) == 3); + assert(_PyOpcode_num_pushed(opcode, oparg) == 1); + ref tos = ref_stack_pop(&refs); + ref_stack_pop(&refs); + ref_stack_pop(&refs); + PUSH_REF(tos.instr, tos.local); + break; + } + case SET_FUNCTION_ATTRIBUTE: { assert(_PyOpcode_num_popped(opcode, oparg) == 2); assert(_PyOpcode_num_pushed(opcode, oparg) == 1); diff --git a/Python/gc.c b/Python/gc.c index 2f373dcb402df3..284ac725d37ac6 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -177,6 +177,11 @@ _PyGC_Init(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; + gcstate->generation_stats = PyMem_RawCalloc(1, sizeof(struct gc_stats)); + if (gcstate->generation_stats == NULL) { + return _PyStatus_NO_MEMORY(); + } + gcstate->garbage = PyList_New(0); if (gcstate->garbage == NULL) { return _PyStatus_NO_MEMORY(); @@ -525,12 +530,19 @@ update_refs(PyGC_Head *containers) return candidates; } +struct visit_decref_context { + PyObject *parent; + struct gc_generation_stats *stats; +}; + /* A traversal callback for subtract_refs. */ static int -visit_decref(PyObject *op, void *parent) +visit_decref(PyObject *op, void *arg) { OBJECT_STAT_INC(object_visits); - _PyObject_ASSERT(_PyObject_CAST(parent), !_PyObject_IsFreed(op)); + struct visit_decref_context *ctx = (struct visit_decref_context *)arg; + ctx->stats->object_visits += 1; + _PyObject_ASSERT(ctx->parent, !_PyObject_IsFreed(op)); if (_PyObject_IS_GC(op)) { PyGC_Head *gc = AS_GC(op); @@ -577,24 +589,35 @@ _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg) * reachable from outside containers, and so can't be collected. */ static void -subtract_refs(PyGC_Head *containers) +subtract_refs(PyGC_Head *containers, struct gc_generation_stats *stats) { traverseproc traverse; PyGC_Head *gc = GC_NEXT(containers); for (; gc != containers; gc = GC_NEXT(gc)) { PyObject *op = FROM_GC(gc); traverse = Py_TYPE(op)->tp_traverse; + struct visit_decref_context ctx = { + .parent = op, + .stats = stats + }; (void) traverse(op, visit_decref, - op); + &ctx); } } +struct visit_reachable_context { + PyGC_Head *head; + struct gc_generation_stats *stats; +}; + /* A traversal callback for move_unreachable. */ static int visit_reachable(PyObject *op, void *arg) { - PyGC_Head *reachable = arg; + struct visit_reachable_context *ctx = (struct visit_reachable_context *)arg; + ctx->stats->object_visits += 1; + PyGC_Head *reachable = ctx->head; OBJECT_STAT_INC(object_visits); if (!_PyObject_IS_GC(op)) { return 0; @@ -667,7 +690,7 @@ visit_reachable(PyObject *op, void *arg) * So we can not gc_list_* functions for unreachable until we remove the flag. */ static void -move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) +move_unreachable(PyGC_Head *young, PyGC_Head *unreachable, struct gc_generation_stats *stats) { // previous elem in the young list, used for restore gc_prev. PyGC_Head *prev = young; @@ -682,6 +705,11 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) * or to the right have been scanned yet. */ + struct visit_reachable_context ctx = { + .head = young, + .stats = stats + }; + validate_consistent_old_space(young); /* Record which old space we are in, and set NEXT_MASK_UNREACHABLE bit for convenience */ uintptr_t flags = NEXT_MASK_UNREACHABLE | (gc->_gc_next & _PyGC_NEXT_MASK_OLD_SPACE_1); @@ -703,7 +731,7 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) // young->_gc_prev == gc. Don't do gc = GC_NEXT(gc) before! (void) traverse(op, visit_reachable, - (void *)young); + &ctx); // relink gc_prev to prev element. _PyGCHead_SET_PREV(gc, prev); // gc is not COLLECTING state after here. @@ -831,7 +859,9 @@ clear_unreachable_mask(PyGC_Head *unreachable) static int visit_move(PyObject *op, void *arg) { - PyGC_Head *tolist = arg; + struct visit_reachable_context *ctx = (struct visit_reachable_context *)arg; + PyGC_Head *tolist = ctx->head; + ctx->stats->object_visits += 1; OBJECT_STAT_INC(object_visits); if (_PyObject_IS_GC(op)) { PyGC_Head *gc = AS_GC(op); @@ -847,8 +877,12 @@ visit_move(PyObject *op, void *arg) * into finalizers set. */ static void -move_legacy_finalizer_reachable(PyGC_Head *finalizers) +move_legacy_finalizer_reachable(PyGC_Head *finalizers, struct gc_generation_stats *stats) { + struct visit_reachable_context ctx = { + .head = finalizers, + .stats = stats + }; traverseproc traverse; PyGC_Head *gc = GC_NEXT(finalizers); for (; gc != finalizers; gc = GC_NEXT(gc)) { @@ -856,7 +890,7 @@ move_legacy_finalizer_reachable(PyGC_Head *finalizers) traverse = Py_TYPE(FROM_GC(gc))->tp_traverse; (void) traverse(FROM_GC(gc), visit_move, - (void *)finalizers); + &ctx); } } @@ -1244,7 +1278,7 @@ flag is cleared (for example, by using 'clear_unreachable_mask' function or by a call to 'move_legacy_finalizers'), the 'unreachable' list is not a normal list and we can not use most gc_list_* functions for it. */ static inline Py_ssize_t -deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) { +deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable, struct gc_generation_stats *stats) { validate_list(base, collecting_clear_unreachable_clear); /* Using ob_refcnt and gc_refs, calculate which objects in the * container set are reachable from outside the set (i.e., have a @@ -1252,7 +1286,7 @@ deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) { * set are taken into account). */ Py_ssize_t candidates = update_refs(base); // gc_prev is used for gc_refs - subtract_refs(base); + subtract_refs(base, stats); /* Leave everything reachable from outside base in base, and move * everything else (in base) to unreachable. @@ -1289,7 +1323,7 @@ deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) { * the reachable objects instead. But this is a one-time cost, probably not * worth complicating the code to speed just a little. */ - move_unreachable(base, unreachable); // gc_prev is pointer again + move_unreachable(base, unreachable, stats); // gc_prev is pointer again validate_list(base, collecting_clear_unreachable_clear); validate_list(unreachable, collecting_set_unreachable_set); return candidates; @@ -1310,7 +1344,8 @@ PREV_MARK_COLLECTING set, but the objects in this set are going to be removed so we can skip the expense of clearing the flag to avoid extra iteration. */ static inline void handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head* still_unreachable, - PyGC_Head *old_generation) + PyGC_Head *old_generation, + struct gc_generation_stats *stats) { // Remove the PREV_MASK_COLLECTING from unreachable // to prepare it for a new call to 'deduce_unreachable' @@ -1320,7 +1355,7 @@ handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head* still_unreachable, // have the PREV_MARK_COLLECTING set, but the objects are going to be // removed so we can skip the expense of clearing the flag. PyGC_Head* resurrected = unreachable; - deduce_unreachable(resurrected, still_unreachable); + deduce_unreachable(resurrected, still_unreachable, stats); clear_unreachable_mask(still_unreachable); // Move the resurrected objects to the old generation for future collection. @@ -1331,7 +1366,7 @@ static void gc_collect_region(PyThreadState *tstate, PyGC_Head *from, PyGC_Head *to, - struct gc_collection_stats *stats); + struct gc_generation_stats *stats); static inline Py_ssize_t gc_list_set_space(PyGC_Head *list, int space) @@ -1364,26 +1399,72 @@ gc_list_set_space(PyGC_Head *list, int space) * scans objects at 1% of the heap size */ #define SCAN_RATE_DIVISOR 10 +static struct gc_generation_stats * +gc_get_stats(GCState *gcstate, int gen) +{ + if (gen == 0) { + struct gc_young_stats_buffer *buffer = &gcstate->generation_stats->young; + buffer->index = (buffer->index + 1) % GC_YOUNG_STATS_SIZE; + struct gc_generation_stats *stats = &buffer->items[buffer->index]; + return stats; + } + else { + struct gc_old_stats_buffer *buffer = &gcstate->generation_stats->old[gen - 1]; + buffer->index = (buffer->index + 1) % GC_OLD_STATS_SIZE; + struct gc_generation_stats *stats = &buffer->items[buffer->index]; + return stats; + } +} + +static struct gc_generation_stats * +gc_get_prev_stats(GCState *gcstate, int gen) +{ + if (gen == 0) { + struct gc_young_stats_buffer *buffer = &gcstate->generation_stats->young; + struct gc_generation_stats *stats = &buffer->items[buffer->index]; + return stats; + } + else { + struct gc_old_stats_buffer *buffer = &gcstate->generation_stats->old[gen - 1]; + struct gc_generation_stats *stats = &buffer->items[buffer->index]; + return stats; + } +} + static void -add_stats(GCState *gcstate, int gen, struct gc_collection_stats *stats) +add_stats(GCState *gcstate, int gen, struct gc_generation_stats *stats) { - gcstate->generation_stats[gen].duration += stats->duration; - gcstate->generation_stats[gen].collected += stats->collected; - gcstate->generation_stats[gen].uncollectable += stats->uncollectable; - gcstate->generation_stats[gen].candidates += stats->candidates; - gcstate->generation_stats[gen].collections += 1; + struct gc_generation_stats *prev_stats = gc_get_prev_stats(gcstate, gen); + struct gc_generation_stats *cur_stats = gc_get_stats(gcstate, gen); + + memcpy(cur_stats, prev_stats, sizeof(struct gc_generation_stats)); + + cur_stats->ts_start = stats->ts_start; + cur_stats->ts_stop = stats->ts_stop; + cur_stats->heap_size = stats->heap_size; + cur_stats->work_to_do = stats->work_to_do; + + cur_stats->collections += 1; + cur_stats->object_visits += stats->object_visits; + cur_stats->collected += stats->collected; + cur_stats->uncollectable += stats->uncollectable; + cur_stats->candidates += stats->candidates; + + cur_stats->objects_transitively_reachable += stats->objects_transitively_reachable; + cur_stats->objects_not_transitively_reachable += stats->objects_not_transitively_reachable; + + cur_stats->duration += stats->duration; } static void gc_collect_young(PyThreadState *tstate, - struct gc_collection_stats *stats) + struct gc_generation_stats *stats) { GCState *gcstate = &tstate->interp->gc; validate_spaces(gcstate); PyGC_Head *young = &gcstate->young.head; PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; untrack_tuples(young); - GC_STAT_ADD(0, collections, 1); PyGC_Head survivors; gc_list_init(&survivors); @@ -1409,6 +1490,7 @@ struct container_and_flag { PyGC_Head *container; int visited_space; intptr_t size; + struct gc_generation_stats *stats; }; /* A traversal callback for adding to container) */ @@ -1417,6 +1499,7 @@ visit_add_to_container(PyObject *op, void *arg) { OBJECT_STAT_INC(object_visits); struct container_and_flag *cf = (struct container_and_flag *)arg; + cf->stats->object_visits += 1; int visited = cf->visited_space; assert(visited == get_gc_state()->visited_space); if (!_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { @@ -1432,12 +1515,16 @@ visit_add_to_container(PyObject *op, void *arg) } static intptr_t -expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCState *gcstate) +expand_region_transitively_reachable(PyGC_Head *container, + PyGC_Head *gc, + GCState *gcstate, + struct gc_generation_stats *stats) { struct container_and_flag arg = { .container = container, .visited_space = gcstate->visited_space, - .size = 0 + .size = 0, + .stats = stats }; assert(GC_NEXT(gc) == container); while (gc != container) { @@ -1506,13 +1593,14 @@ move_to_reachable(PyObject *op, PyGC_Head *reachable, int visited_space) } static intptr_t -mark_all_reachable(PyGC_Head *reachable, PyGC_Head *visited, int visited_space) +mark_all_reachable(PyGC_Head *reachable, PyGC_Head *visited, int visited_space, struct gc_generation_stats *stats) { // Transitively traverse all objects from reachable, until empty struct container_and_flag arg = { .container = reachable, .visited_space = visited_space, - .size = 0 + .size = 0, + .stats = stats }; while (!gc_list_is_empty(reachable)) { PyGC_Head *gc = _PyGCHead_NEXT(reachable); @@ -1529,7 +1617,7 @@ mark_all_reachable(PyGC_Head *reachable, PyGC_Head *visited, int visited_space) } static intptr_t -mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, bool start) +mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, bool start, struct gc_generation_stats *stats) { PyGC_Head reachable; gc_list_init(&reachable); @@ -1582,13 +1670,13 @@ mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, b ts = PyThreadState_Next(ts); HEAD_UNLOCK(runtime); } - objects_marked += mark_all_reachable(&reachable, visited, visited_space); + objects_marked += mark_all_reachable(&reachable, visited, visited_space, stats); assert(gc_list_is_empty(&reachable)); return objects_marked; } static intptr_t -mark_global_roots(PyInterpreterState *interp, PyGC_Head *visited, int visited_space) +mark_global_roots(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, struct gc_generation_stats *stats) { PyGC_Head reachable; gc_list_init(&reachable); @@ -1605,19 +1693,19 @@ mark_global_roots(PyInterpreterState *interp, PyGC_Head *visited, int visited_sp objects_marked += move_to_reachable(types->for_extensions.initialized[i].tp_dict, &reachable, visited_space); objects_marked += move_to_reachable(types->for_extensions.initialized[i].tp_subclasses, &reachable, visited_space); } - objects_marked += mark_all_reachable(&reachable, visited, visited_space); + objects_marked += mark_all_reachable(&reachable, visited, visited_space, stats); assert(gc_list_is_empty(&reachable)); return objects_marked; } static intptr_t -mark_at_start(PyThreadState *tstate) +mark_at_start(PyThreadState *tstate, struct gc_generation_stats *stats) { // TO DO -- Make this incremental GCState *gcstate = &tstate->interp->gc; PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; - Py_ssize_t objects_marked = mark_global_roots(tstate->interp, visited, gcstate->visited_space); - objects_marked += mark_stacks(tstate->interp, visited, gcstate->visited_space, true); + Py_ssize_t objects_marked = mark_global_roots(tstate->interp, visited, gcstate->visited_space, stats); + objects_marked += mark_stacks(tstate->interp, visited, gcstate->visited_space, true, stats); gcstate->work_to_do -= objects_marked; gcstate->phase = GC_PHASE_COLLECT; validate_spaces(gcstate); @@ -1654,9 +1742,8 @@ assess_work_to_do(GCState *gcstate) } static void -gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) +gc_collect_increment(PyThreadState *tstate, struct gc_generation_stats *stats) { - GC_STAT_ADD(1, collections, 1); GCState *gcstate = &tstate->interp->gc; gcstate->work_to_do += assess_work_to_do(gcstate); if (gcstate->work_to_do < 0) { @@ -1664,10 +1751,10 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) } untrack_tuples(&gcstate->young.head); if (gcstate->phase == GC_PHASE_MARK) { - Py_ssize_t objects_marked = mark_at_start(tstate); - GC_STAT_ADD(1, objects_transitively_reachable, objects_marked); - gcstate->work_to_do -= objects_marked; + Py_ssize_t objects_marked = mark_at_start(tstate, stats); + stats->objects_transitively_reachable += objects_marked; stats->candidates += objects_marked; + gcstate->work_to_do -= objects_marked; validate_spaces(gcstate); return; } @@ -1679,8 +1766,8 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) if (scale_factor < 2) { scale_factor = 2; } - intptr_t objects_marked = mark_stacks(tstate->interp, visited, gcstate->visited_space, false); - GC_STAT_ADD(1, objects_transitively_reachable, objects_marked); + intptr_t objects_marked = mark_stacks(tstate->interp, visited, gcstate->visited_space, false, stats); + stats->objects_transitively_reachable += objects_marked; gcstate->work_to_do -= objects_marked; gc_list_set_space(&gcstate->young.head, gcstate->visited_space); gc_list_merge(&gcstate->young.head, &increment); @@ -1695,9 +1782,9 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) increment_size++; assert(!_Py_IsImmortal(FROM_GC(gc))); gc_set_old_space(gc, gcstate->visited_space); - increment_size += expand_region_transitively_reachable(&increment, gc, gcstate); + increment_size += expand_region_transitively_reachable(&increment, gc, gcstate, stats); } - GC_STAT_ADD(1, objects_not_transitively_reachable, increment_size); + stats->objects_not_transitively_reachable += increment_size; validate_list(&increment, collecting_clear_unreachable_clear); gc_list_validate_space(&increment, gcstate->visited_space); PyGC_Head survivors; @@ -1715,9 +1802,8 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) static void gc_collect_full(PyThreadState *tstate, - struct gc_collection_stats *stats) + struct gc_generation_stats *stats) { - GC_STAT_ADD(2, collections, 1); GCState *gcstate = &tstate->interp->gc; validate_spaces(gcstate); PyGC_Head *young = &gcstate->young.head; @@ -1749,7 +1835,7 @@ static void gc_collect_region(PyThreadState *tstate, PyGC_Head *from, PyGC_Head *to, - struct gc_collection_stats *stats) + struct gc_generation_stats *stats) { PyGC_Head unreachable; /* non-problematic unreachable trash */ PyGC_Head finalizers; /* objects with, & reachable from, __del__ */ @@ -1760,7 +1846,7 @@ gc_collect_region(PyThreadState *tstate, assert(!_PyErr_Occurred(tstate)); gc_list_init(&unreachable); - stats->candidates = deduce_unreachable(from, &unreachable); + stats->candidates = deduce_unreachable(from, &unreachable, stats); validate_consistent_old_space(from); untrack_tuples(from); @@ -1782,7 +1868,7 @@ gc_collect_region(PyThreadState *tstate, * unreachable objects reachable *from* those are also uncollectable, * and we move those into the finalizers list too. */ - move_legacy_finalizer_reachable(&finalizers); + move_legacy_finalizer_reachable(&finalizers, stats); validate_list(&finalizers, collecting_clear_unreachable_clear); validate_list(&unreachable, collecting_set_unreachable_clear); /* Print debugging information. */ @@ -1805,7 +1891,7 @@ gc_collect_region(PyThreadState *tstate, * objects that are still unreachable */ PyGC_Head final_unreachable; gc_list_init(&final_unreachable); - handle_resurrected_objects(&unreachable, &final_unreachable, to); + handle_resurrected_objects(&unreachable, &final_unreachable, to, stats); /* Clear weakrefs to objects in the unreachable set. See the comments * above handle_weakref_callbacks() for details. @@ -1842,7 +1928,7 @@ gc_collect_region(PyThreadState *tstate, */ static void do_gc_callback(GCState *gcstate, const char *phase, - int generation, struct gc_collection_stats *stats) + int generation, struct gc_generation_stats *stats) { assert(!PyErr_Occurred()); @@ -1890,7 +1976,7 @@ do_gc_callback(GCState *gcstate, const char *phase, static void invoke_gc_callback(GCState *gcstate, const char *phase, - int generation, struct gc_collection_stats *stats) + int generation, struct gc_generation_stats *stats) { if (gcstate->callbacks == NULL) { return; @@ -2082,7 +2168,7 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) } gcstate->frame = tstate->current_frame; - struct gc_collection_stats stats = { 0 }; + struct gc_generation_stats stats = { 0 }; if (reason != _Py_GC_REASON_SHUTDOWN) { invoke_gc_callback(gcstate, "start", generation, &stats); } @@ -2093,8 +2179,9 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) if (PyDTrace_GC_START_ENABLED()) { PyDTrace_GC_START(generation); } - PyTime_t start, stop; - (void)PyTime_PerfCounterRaw(&start); + stats.heap_size = gcstate->heap_size; + stats.work_to_do = gcstate->work_to_do; + (void)PyTime_PerfCounterRaw(&stats.ts_start); PyObject *exc = _PyErr_GetRaisedException(tstate); switch(generation) { case 0: @@ -2109,8 +2196,8 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) default: Py_UNREACHABLE(); } - (void)PyTime_PerfCounterRaw(&stop); - stats.duration = PyTime_AsSecondsDouble(stop - start); + (void)PyTime_PerfCounterRaw(&stats.ts_stop); + stats.duration = PyTime_AsSecondsDouble(stats.ts_stop - stats.ts_start); add_stats(gcstate, generation, &stats); if (PyDTrace_GC_DONE_ENABLED()) { PyDTrace_GC_DONE(stats.uncollectable + stats.collected); @@ -2217,6 +2304,8 @@ _PyGC_Fini(PyInterpreterState *interp) GCState *gcstate = &interp->gc; Py_CLEAR(gcstate->garbage); Py_CLEAR(gcstate->callbacks); + PyMem_RawFree(gcstate->generation_stats); + gcstate->generation_stats = NULL; /* Prevent a subtle bug that affects sub-interpreters that use basic * single-phase init extensions (m_size == -1). Those extensions cause objects diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 0ec9c58a792e6d..4b46ca04f56b20 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1698,6 +1698,11 @@ _PyGC_Init(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; + gcstate->generation_stats = PyMem_RawCalloc(1, sizeof(struct gc_stats)); + if (gcstate->generation_stats == NULL) { + return _PyStatus_NO_MEMORY(); + } + gcstate->garbage = PyList_New(0); if (gcstate->garbage == NULL) { return _PyStatus_NO_MEMORY(); @@ -2383,6 +2388,21 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, handle_legacy_finalizers(state); } +static struct gc_generation_stats * +get_stats(GCState *gcstate, int gen) +{ + if (gen == 0) { + struct gc_young_stats_buffer *buffer = &gcstate->generation_stats->young; + struct gc_generation_stats *stats = &buffer->items[buffer->index]; + return stats; + } + else { + struct gc_old_stats_buffer *buffer = &gcstate->generation_stats->old[gen - 1]; + struct gc_generation_stats *stats = &buffer->items[buffer->index]; + return stats; + } +} + /* This is the main function. Read this to understand how the * collection process works. */ static Py_ssize_t @@ -2471,7 +2491,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) } /* Update stats */ - struct gc_generation_stats *stats = &gcstate->generation_stats[generation]; + struct gc_generation_stats *stats = get_stats(gcstate, generation); stats->collections++; stats->collected += m; stats->uncollectable += n; @@ -2816,6 +2836,8 @@ _PyGC_Fini(PyInterpreterState *interp) GCState *gcstate = &interp->gc; Py_CLEAR(gcstate->garbage); Py_CLEAR(gcstate->callbacks); + PyMem_RawFree(gcstate->generation_stats); + gcstate->generation_stats = NULL; /* We expect that none of this interpreters objects are shared with other interpreters. diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0d8686c1b5856a..127a45ef591053 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2348,17 +2348,9 @@ _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST + // _GUARD_CALLABLE_BUILTIN_FAST { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2370,6 +2362,17 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CALL_BUILTIN_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _Py_BuiltinCallFast_StackRefSteal( @@ -2417,17 +2420,9 @@ _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST_WITH_KEYWORDS + // _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2439,6 +2434,17 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CALL_BUILTIN_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRefSteal(callable, arguments, total_args); @@ -2485,32 +2491,38 @@ _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_BUILTIN_O + // _GUARD_CALLABLE_BUILTIN_O { - args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - if (total_args != 1) { + if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (!PyCFunction_CheckExact(callable_o)) { + if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CALL_BUILTIN_O + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyStackRef_IsNull(self_or_null)) { + args--; + } if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); @@ -3021,31 +3033,44 @@ _PyStackRef value2_st; _PyStackRef value1_st; _PyStackRef res; - value1_st = stack_pointer[-1]; - value2_st = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); - PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); - PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - _PyStackRef tmp = value1_st; - value1_st = PyStackRef_NULL; - stack_pointer[-1] = value1_st; - PyStackRef_CLOSE(tmp); - tmp = value2_st; - value2_st = PyStackRef_NULL; - stack_pointer[-2] = value2_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (res_o == NULL) { - JUMP_TO_LABEL(error); + _PyStackRef vs1; + _PyStackRef vs2; + _PyStackRef value; + // _CALL_INTRINSIC_2 + { + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + vs1 = value1_st; + vs2 = value2_st; + } + // _POP_TOP + { + value = vs2; + stack_pointer[-2] = res; + stack_pointer[-1] = vs1; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = vs1; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -3754,47 +3779,58 @@ _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST + // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (total_args == 0) { + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + if (method->d_method->ml_flags != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_FASTCALL) { + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args == 0) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - assert(self != NULL); + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CALL_METHOD_DESCRIPTOR_FAST + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFast cfunc = _PyCFunctionFast_CAST(method->d_method->ml_meth); PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( callable, - meth, + cfunc, self, arguments, total_args @@ -3839,12 +3875,23 @@ _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (method->d_method->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { @@ -3856,31 +3903,31 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + } + // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; } - PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); assert(self != NULL); - if (!Py_IS_TYPE(self, d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords_CAST(method->d_method->ml_meth); PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( callable, - meth, + cfunc, self, arguments, total_args @@ -3925,49 +3972,60 @@ _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_NOARGS + // _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - if (total_args != 1) { + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + if (method->d_method->ml_flags != METH_NOARGS) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDef *meth = method->d_method; - _PyStackRef self_stackref = args[0]; - PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - if (!Py_IS_TYPE(self, method->d_common.d_type)) { + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (meth->ml_flags != METH_NOARGS) { + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CHECK_RECURSION_LIMIT + { if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CALL_METHOD_DESCRIPTOR_NOARGS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + assert(oparg == 1 || !PyStackRef_IsNull(self_or_null)); + if (!PyStackRef_IsNull(self_or_null)) { + args--; + } + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; + PyCFunction cfunc = method->d_method->ml_meth; _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4022,54 +4080,62 @@ _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_O + // _GUARD_CALLABLE_METHOD_DESCRIPTOR_O { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (total_args != 2) { + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + if (method->d_method->ml_flags != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_O) { + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args != 2) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (_Py_ReachedRecursionLimit(tstate)) { + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - _PyStackRef arg_stackref = arguments[1]; - _PyStackRef self_stackref = arguments[0]; - if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type)) { + } + // _CHECK_RECURSION_LIMIT + { + if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CALL_METHOD_DESCRIPTOR_O + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + } STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; + PyCFunction cfunc = method->d_method->ml_meth; + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + PyObject *arg = PyStackRef_AsPyObjectBorrow(arguments[1]); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, - PyStackRef_AsPyObjectBorrow(self_stackref), - PyStackRef_AsPyObjectBorrow(arg_stackref)); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, arg); stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -4740,13 +4806,16 @@ next_instr += 1; INSTRUCTION_STATS(CLEANUP_THROW); _PyStackRef sub_iter; + _PyStackRef null_in; _PyStackRef last_sent_val; _PyStackRef exc_value_st; _PyStackRef none; + _PyStackRef null_out; _PyStackRef value; exc_value_st = stack_pointer[-1]; last_sent_val = stack_pointer[-2]; - sub_iter = stack_pointer[-3]; + null_in = stack_pointer[-3]; + sub_iter = stack_pointer[-4]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); #if !_Py_TAIL_CALL_INTERP assert(throwflag); @@ -4760,7 +4829,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = sub_iter; sub_iter = value; - stack_pointer[-3] = sub_iter; + stack_pointer[-4] = sub_iter; PyStackRef_CLOSE(tmp); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; @@ -4770,9 +4839,14 @@ last_sent_val = PyStackRef_NULL; stack_pointer[-2] = last_sent_val; PyStackRef_CLOSE(tmp); + tmp = null_in; + null_in = PyStackRef_NULL; + stack_pointer[-3] = null_in; + PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; + stack_pointer += -4; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + null_out = null_in; none = PyStackRef_None; } else { @@ -4782,8 +4856,9 @@ JUMP_TO_LABEL(exception_unwind); } stack_pointer[0] = none; - stack_pointer[1] = value; - stack_pointer += 2; + stack_pointer[1] = null_out; + stack_pointer[2] = value; + stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5526,31 +5601,38 @@ _PyStackRef callable; _PyStackRef dict; _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - callable = stack_pointer[-5 - (oparg - 1)]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_MergeEx(dict_o, update_o, 2); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { + _PyStackRef u; + _PyStackRef value; + // _DICT_MERGE + { + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + PyObject *dupkey = NULL; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatKwargsError(tstate, callable_o, update_o); + int err = _PyDict_MergeUniq(dict_o, update_o, &dupkey); stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatKwargsError(tstate, callable_o, update_o, dupkey); + Py_XDECREF(dupkey); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + u = update; + } + // _POP_TOP + { + value = u; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); } - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -5564,36 +5646,53 @@ INSTRUCTION_STATS(DICT_UPDATE); _PyStackRef dict; _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Update(dict_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { + _PyStackRef upd; + _PyStackRef value; + // _DICT_UPDATE + { + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + int err = PyDict_Update(dict_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { + if (err < 0) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update_o)->tp_name); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *exc = _PyErr_GetRaisedException(tstate); + int has_keys = PyObject_HasAttrWithError(update_o, &_Py_ID(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (has_keys == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'%T' object is not a mapping", + update_o); + Py_DECREF(exc); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_ChainExceptions1(exc); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + JUMP_TO_LABEL(error); } + upd = update; + } + // _POP_TOP + { + value = upd; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); } - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -5668,13 +5767,16 @@ next_instr += 1; INSTRUCTION_STATS(END_SEND); _PyStackRef receiver; + _PyStackRef index_or_null; _PyStackRef value; _PyStackRef val; value = stack_pointer[-1]; - receiver = stack_pointer[-2]; + index_or_null = stack_pointer[-2]; + receiver = stack_pointer[-3]; val = value; - stack_pointer[-2] = val; - stack_pointer += -1; + (void)index_or_null; + stack_pointer[-3] = val; + stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(receiver); @@ -6294,31 +6396,15 @@ _PyStackRef index_or_null; iterable = stack_pointer[-1]; #ifdef Py_STATS - _PyFrame_SetStackPointer(frame, stack_pointer); _Py_GatherStats_GetIter(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); #endif - PyTypeObject *tp = PyStackRef_TYPE(iterable); - if (tp == &PyTuple_Type || tp == &PyList_Type) { - iter = iterable; - index_or_null = PyStackRef_TagInt(0); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - index_or_null = PyStackRef_NULL; - stack_pointer += 1; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef result = _PyEval_GetIter(iterable, &index_or_null, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsError(result)) { + JUMP_TO_LABEL(pop_1_error); } + iter = result; stack_pointer[-1] = iter; stack_pointer[0] = index_or_null; stack_pointer += 1; @@ -6354,51 +6440,6 @@ DISPATCH(); } - TARGET(GET_YIELD_FROM_ITER) { - #if _Py_TAIL_CALL_INTERP - int opcode = GET_YIELD_FROM_ITER; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_YIELD_FROM_ITER); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); - if (PyCoro_CheckExact(iterable_o)) { - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); - } - iter = iterable; - } - else if (PyGen_CheckExact(iterable_o)) { - iter = iterable; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(iterable_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = iterable; - iterable = iter; - stack_pointer[-1] = iterable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer[-1] = iter; - DISPATCH(); - } - TARGET(IMPORT_FROM) { #if _Py_TAIL_CALL_INTERP int opcode = IMPORT_FROM; @@ -7002,10 +7043,12 @@ next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_SEND); _PyStackRef receiver; + _PyStackRef index_or_null; _PyStackRef value; _PyStackRef val; value = stack_pointer[-1]; - receiver = stack_pointer[-2]; + index_or_null = stack_pointer[-2]; + receiver = stack_pointer[-3]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7016,8 +7059,9 @@ } } val = value; - stack_pointer[-2] = val; - stack_pointer += -1; + (void)index_or_null; + stack_pointer[-3] = val; + stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(receiver); @@ -7911,40 +7955,45 @@ INSTRUCTION_STATS(LIST_EXTEND); _PyStackRef list_st; _PyStackRef iterable_st; - iterable_st = stack_pointer[-1]; - list_st = stack_pointer[-2 - (oparg-1)]; - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (none_val == NULL) { + _PyStackRef i; + _PyStackRef value; + // _LIST_EXTEND + { + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { + if (none_val == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + JUMP_TO_LABEL(error); } + assert(Py_IsNone(none_val)); + i = iterable_st; + } + // _POP_TOP + { + value = i; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable_st); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); } - assert(Py_IsNone(none_val)); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable_st); - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -10733,11 +10782,12 @@ _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef receiver; + _PyStackRef null_or_index; _PyStackRef v; _PyStackRef retval; // _SPECIALIZE_SEND { - receiver = stack_pointer[-2]; + receiver = stack_pointer[-3]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION @@ -10755,6 +10805,7 @@ // _SEND { v = stack_pointer[-1]; + null_or_index = stack_pointer[-2]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); PyObject *retval_o; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); @@ -10775,53 +10826,66 @@ gen_frame->previous = frame; DISPATCH_INLINED(gen_frame); } - if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { + if (!PyStackRef_IsNull(null_or_index)) { _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, receiver, &null_or_index); stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + JUMP_TO_LABEL(error); + } + JUMPBY(oparg); + stack_pointer[-2] = null_or_index; + DISPATCH(); + } + retval = item; } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = PyObject_CallMethodOneArg(receiver_o, - &_Py_ID(send), - PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (retval_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { + if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); stack_pointer = _PyFrame_GetStackPointer(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyGen_FetchStopIterationValue(&retval_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err == 0) { - assert(retval_o != NULL); - JUMPBY(oparg); - } else { - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); } + if (retval_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyGen_FetchStopIterationValue(&retval_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err == 0) { + assert(retval_o != NULL); + JUMPBY(oparg); + } + else { + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + } + retval = PyStackRef_FromPyObjectSteal(retval_o); } - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + stack_pointer[-2] = null_or_index; + stack_pointer[-1] = retval; _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); - retval = PyStackRef_FromPyObjectSteal(retval_o); } - stack_pointer[0] = retval; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10852,7 +10916,7 @@ // _SEND_GEN_FRAME { v = stack_pointer[-1]; - receiver = stack_pointer[-2]; + receiver = stack_pointer[-3]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { UPDATE_MISS_STATS(SEND); diff --git a/Python/getargs.c b/Python/getargs.c index 31cd4ad3f652d9..3f423266bff7f4 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -2421,7 +2421,7 @@ vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, if (i < parser->min) { /* Less arguments than required */ if (i < pos) { - Py_ssize_t min = Py_MIN(pos, parser->min); + int min = Py_MIN(pos, parser->min); PyErr_Format(PyExc_TypeError, "%.200s%s takes %s %d positional argument%s" " (%zd given)", @@ -2435,7 +2435,7 @@ vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, else { keyword = PyTuple_GET_ITEM(kwtuple, i - pos); PyErr_Format(PyExc_TypeError, "%.200s%s missing required " - "argument '%U' (pos %d)", + "argument '%U' (pos %zd)", (parser->fname == NULL) ? "function" : parser->fname, (parser->fname == NULL) ? "" : "()", keyword, i+1); @@ -2476,7 +2476,7 @@ vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, /* arg present in tuple and in dict */ PyErr_Format(PyExc_TypeError, "argument for %.200s%s given by name ('%U') " - "and position (%d)", + "and position (%zd)", (parser->fname == NULL) ? "function" : parser->fname, (parser->fname == NULL) ? "" : "()", keyword, i+1); diff --git a/Python/import.c b/Python/import.c index 34224f4c6d6514..e298fbee536c1b 100644 --- a/Python/import.c +++ b/Python/import.c @@ -4377,6 +4377,12 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, PyUnicode_GET_LENGTH(name), -1); if (dot < 0) { + PyObject *lazy_submodules = ensure_lazy_submodules( + (PyDictObject *)lazy_modules, name); + if (lazy_submodules == NULL) { + goto done; + } + Py_DECREF(lazy_submodules); ret = 0; goto done; } @@ -5714,6 +5720,7 @@ imp_module_exec(PyObject *module) static PyModuleDef_Slot imp_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, imp_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Python/initconfig.c b/Python/initconfig.c index caf42f5247c2f2..8dc9602ff13df7 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -511,7 +511,7 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS do { \ obj = (EXPR); \ if (obj == NULL) { \ - return NULL; \ + goto fail; \ } \ int res = PyDict_SetItemString(dict, (KEY), obj); \ Py_DECREF(obj); \ diff --git a/Python/interpconfig.c b/Python/interpconfig.c index 1add8a81425b9a..a37bd3f5b23a01 100644 --- a/Python/interpconfig.c +++ b/Python/interpconfig.c @@ -208,7 +208,7 @@ interp_config_from_dict(PyObject *origdict, PyInterpreterConfig *config, } else if (unused > 0) { PyErr_Format(PyExc_ValueError, - "config dict has %d extra items (%R)", unused, dict); + "config dict has %zd extra items (%R)", unused, dict); goto error; } diff --git a/Python/lock.c b/Python/lock.c index ad97bfd93c8495..752a5899e088a5 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -27,8 +27,10 @@ static const PyTime_t TIME_TO_BE_FAIR_NS = 1000*1000; // enabled. #if Py_GIL_DISABLED static const int MAX_SPIN_COUNT = 40; +static const int RELOAD_SPIN_MASK = 3; #else static const int MAX_SPIN_COUNT = 0; +static const int RELOAD_SPIN_MASK = 1; #endif struct mutex_entry { @@ -79,6 +81,16 @@ _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags) }; Py_ssize_t spin_count = 0; +#ifdef Py_GIL_DISABLED + // Using thread-id as a way of reducing contention further in the reload below. + // It adds a pseudo-random starting offset to the recurrence, so that threads + // are less likely to try and run compare-exchange at the same time. + // The lower bits of platform thread ids are likely to not be random, + // hence the right shift. + const Py_ssize_t tid = (Py_ssize_t)(_Py_ThreadId() >> 12); +#else + const Py_ssize_t tid = 0; +#endif for (;;) { if ((v & _Py_LOCKED) == 0) { // The lock is unlocked. Try to grab it. @@ -92,6 +104,9 @@ _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags) // Spin for a bit. _Py_yield(); spin_count++; + if (((spin_count + tid) & RELOAD_SPIN_MASK) == 0) { + v = _Py_atomic_load_uint8_relaxed(&m->_bits); + } continue; } diff --git a/Python/marshal.c b/Python/marshal.c index cc6a787ba75022..b60a36e128cd9f 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -2135,6 +2135,7 @@ marshal_module_exec(PyObject *mod) } static PyModuleDef_Slot marshalmodule_slots[] = { + _Py_ABI_SLOT, {Py_mod_exec, marshal_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Python/modsupport.c b/Python/modsupport.c index 239c6c6a1b3bfa..bab21d1b2be5b5 100644 --- a/Python/modsupport.c +++ b/Python/modsupport.c @@ -688,9 +688,12 @@ static int _abiinfo_raise(const char *module_name, const char *format, ...) va_list vargs; va_start(vargs, format); if (_PyUnicodeWriter_FormatV(writer, format, vargs) < 0) { + va_end(vargs); PyUnicodeWriter_Discard(writer); return -1; } + + va_end(vargs); PyObject *message = PyUnicodeWriter_Finish(writer); if (!message) { return -1; @@ -732,15 +735,15 @@ int PyABIInfo_Check(PyABIInfo *info, const char *module_name) return _abiinfo_raise( module_name, "incompatible future stable ABI version (%d.%d)", - ((info->abi_version) >> 24) % 0xff, - ((info->abi_version) >> 16) % 0xff); + ((info->abi_version) >> 24) & 0xff, + ((info->abi_version) >> 16) & 0xff); } if (info->abi_version < Py_PACK_VERSION(3, 2)) { return _abiinfo_raise( module_name, "invalid stable ABI version (%d.%d)", - ((info->abi_version) >> 24) % 0xff, - ((info->abi_version) >> 16) % 0xff); + ((info->abi_version) >> 24) & 0xff, + ((info->abi_version) >> 16) & 0xff); } } if (info->flags & PyABIInfo_INTERNAL) { @@ -755,8 +758,8 @@ int PyABIInfo_Check(PyABIInfo *info, const char *module_name) return _abiinfo_raise( module_name, "incompatible ABI version (%d.%d)", - ((info->abi_version) >> 24) % 0xff, - ((info->abi_version) >> 16) % 0xff); + ((info->abi_version) >> 24) & 0xff, + ((info->abi_version) >> 16) & 0xff); } } } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index def462bacec176..48fe9c14f4e2dd 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -16,10 +16,8 @@ static void *opcode_targets_table[256] = { &&TARGET_FORMAT_WITH_SPEC, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, - &&TARGET_GET_ITER, - &&TARGET_RESERVED, &&TARGET_GET_LEN, - &&TARGET_GET_YIELD_FROM_ITER, + &&TARGET_RESERVED, &&TARGET_INTERPRETER_EXIT, &&TARGET_LOAD_BUILD_CLASS, &&TARGET_LOAD_LOCALS, @@ -72,6 +70,7 @@ static void *opcode_targets_table[256] = { &&TARGET_EXTENDED_ARG, &&TARGET_FOR_ITER, &&TARGET_GET_AWAITABLE, + &&TARGET_GET_ITER, &&TARGET_IMPORT_FROM, &&TARGET_IMPORT_NAME, &&TARGET_IS_OP, @@ -128,6 +127,7 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -379,7 +379,7 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&TARGET_TRACE_RECORD, + &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -626,7 +626,6 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); @@ -868,7 +867,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [GET_AWAITABLE] = _TAIL_CALL_GET_AWAITABLE, [GET_ITER] = _TAIL_CALL_GET_ITER, [GET_LEN] = _TAIL_CALL_GET_LEN, - [GET_YIELD_FROM_ITER] = _TAIL_CALL_GET_YIELD_FROM_ITER, [IMPORT_FROM] = _TAIL_CALL_IMPORT_FROM, [IMPORT_NAME] = _TAIL_CALL_IMPORT_NAME, [INSTRUMENTED_CALL] = _TAIL_CALL_INSTRUMENTED_CALL, @@ -1002,6 +1000,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, + [120] = _TAIL_CALL_UNKNOWN_OPCODE, [121] = _TAIL_CALL_UNKNOWN_OPCODE, [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1126,7 +1125,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [GET_AWAITABLE] = _TAIL_CALL_TRACE_RECORD, [GET_ITER] = _TAIL_CALL_TRACE_RECORD, [GET_LEN] = _TAIL_CALL_TRACE_RECORD, - [GET_YIELD_FROM_ITER] = _TAIL_CALL_TRACE_RECORD, [IMPORT_FROM] = _TAIL_CALL_TRACE_RECORD, [IMPORT_NAME] = _TAIL_CALL_TRACE_RECORD, [INSTRUMENTED_CALL] = _TAIL_CALL_TRACE_RECORD, @@ -1260,6 +1258,7 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACE_RECORD, [WITH_EXCEPT_START] = _TAIL_CALL_TRACE_RECORD, [YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, + [120] = _TAIL_CALL_UNKNOWN_OPCODE, [121] = _TAIL_CALL_UNKNOWN_OPCODE, [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 520420e9878575..92e1c081d524db 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -30,6 +30,7 @@ #include "pycore_unicodeobject.h" #include "pycore_ceval.h" #include "pycore_floatobject.h" +#include "pycore_setobject.h" #include #include @@ -251,6 +252,7 @@ add_op(JitOptContext *ctx, _PyUOpInstruction *this_instr, #define sym_is_not_null _Py_uop_sym_is_not_null #define sym_is_const _Py_uop_sym_is_const #define sym_is_safe_const _Py_uop_sym_is_safe_const +#define sym_is_not_container _Py_uop_sym_is_not_container #define sym_get_const _Py_uop_sym_get_const #define sym_new_const_steal _Py_uop_sym_new_const_steal #define sym_get_const_as_stackref _Py_uop_sym_get_const_as_stackref @@ -348,6 +350,33 @@ optimize_to_bool( return 0; } +static void +optimize_dict_known_hash( + JitOptContext *ctx, _PyBloomFilter *dependencies, _PyUOpInstruction *this_instr, + PyObject *sub, uint16_t opcode) +{ + if (PyUnicode_CheckExact(sub) || PyLong_CheckExact(sub) || PyBytes_CheckExact(sub) + || PyFloat_CheckExact(sub) || PyComplex_CheckExact(sub)) { + // PyObject_Hash can't fail on these types + ADD_OP(opcode, 0, PyObject_Hash(sub)); + } + else if (PyTuple_CheckExact(sub)) { + // only use known hash variant when hash of tuple is already computed + // since computing it can call arbitrary code + Py_hash_t hash = ((PyTupleObject *)sub)->ob_hash; + if (hash != -1) { + ADD_OP(opcode, 0, hash); + } + } + else if (Py_TYPE(sub)->tp_hash == PyBaseObject_Type.tp_hash) { + // for user-defined objects which don't override tp_hash + Py_hash_t hash = PyObject_Hash(sub); + ADD_OP(opcode, 0, hash); + PyType_Watch(TYPE_WATCHER_ID, (PyObject *)Py_TYPE(sub)); + _Py_BloomFilter_Add(dependencies, Py_TYPE(sub)); + } +} + static void eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, bool exit) { @@ -367,7 +396,11 @@ lookup_attr(JitOptContext *ctx, _PyBloomFilter *dependencies, _PyUOpInstruction if (type && PyType_Check(type)) { PyObject *lookup = _PyType_Lookup(type, name); if (lookup) { - int opcode = _Py_IsImmortal(lookup) ? immortal : mortal; + int opcode = mortal; + // if the object is immortal or the type is immutable, borrowing is safe + if (_Py_IsImmortal(lookup) || (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE)) { + opcode = immortal; + } ADD_OP(opcode, 0, (uintptr_t)lookup); PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 2364bf75d104de..b8148ef57ede0c 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -93,7 +93,19 @@ dummy_func(void) { if (sym_is_immortal(PyJitRef_Unwrap(value))) { ADD_OP(_NOP, 0, 0); } - value = PyJitRef_StripReferenceInfo(value); + value = PyJitRef_StripBorrowInfo(value); + } + + op(_COPY_FREE_VARS, (--)) { + PyCodeObject *co = get_current_code_object(ctx); + if (co == NULL) { + ctx->done = true; + break; + } + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + ctx->frame->locals[offset + i] = sym_new_not_null(ctx); + } } op(_LOAD_FAST_CHECK, (-- value)) { @@ -102,20 +114,24 @@ dummy_func(void) { if (sym_is_null(value)) { ctx->done = true; } + assert(!PyJitRef_IsUnique(value)); } op(_LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); + assert(!PyJitRef_IsUnique(value)); } op(_LOAD_FAST_BORROW, (-- value)) { value = PyJitRef_Borrow(GETLOCAL(oparg)); + assert(!PyJitRef_IsUnique(value)); } op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); JitOptRef temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; + assert(!PyJitRef_IsUnique(value)); } op(_STORE_ATTR_INSTANCE_VALUE, (offset/1, value, owner -- o)) { @@ -132,7 +148,7 @@ dummy_func(void) { op(_SWAP_FAST, (value -- trash)) { JitOptRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; + GETLOCAL(oparg) = PyJitRef_RemoveUnique(value); trash = tmp; } @@ -149,6 +165,11 @@ dummy_func(void) { } op(_STORE_SUBSCR_DICT, (value, dict_st, sub -- st)) { + PyObject *sub_o = sym_get_const(ctx, sub); + if (sub_o != NULL) { + optimize_dict_known_hash(ctx, dependencies, this_instr, + sub_o, _STORE_SUBSCR_DICT_KNOWN_HASH); + } (void)value; st = dict_st; } @@ -288,42 +309,98 @@ dummy_func(void) { } op(_BINARY_OP_ADD_INT, (left, right -- res, l, r)) { - res = sym_new_compact_int(ctx); + if (PyJitRef_IsUnique(left)) { + REPLACE_OP(this_instr, _BINARY_OP_ADD_INT_INPLACE, 0, 0); + } + else if (PyJitRef_IsUnique(right)) { + REPLACE_OP(this_instr, _BINARY_OP_ADD_INT_INPLACE_RIGHT, 0, 0); + } + // Result may be a unique compact int or a cached small int + // at runtime. Mark as unique; inplace ops verify at runtime. + res = PyJitRef_MakeUnique(sym_new_compact_int(ctx)); l = left; r = right; REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); } op(_BINARY_OP_SUBTRACT_INT, (left, right -- res, l, r)) { - res = sym_new_compact_int(ctx); + if (PyJitRef_IsUnique(left)) { + REPLACE_OP(this_instr, _BINARY_OP_SUBTRACT_INT_INPLACE, 0, 0); + } + else if (PyJitRef_IsUnique(right)) { + REPLACE_OP(this_instr, _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, 0, 0); + } + res = PyJitRef_MakeUnique(sym_new_compact_int(ctx)); l = left; r = right; REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); } op(_BINARY_OP_MULTIPLY_INT, (left, right -- res, l, r)) { - res = sym_new_compact_int(ctx); + if (PyJitRef_IsUnique(left)) { + REPLACE_OP(this_instr, _BINARY_OP_MULTIPLY_INT_INPLACE, 0, 0); + } + else if (PyJitRef_IsUnique(right)) { + REPLACE_OP(this_instr, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, 0, 0); + } + res = PyJitRef_MakeUnique(sym_new_compact_int(ctx)); l = left; r = right; REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); } op(_BINARY_OP_ADD_FLOAT, (left, right -- res, l, r)) { - res = sym_new_type(ctx, &PyFloat_Type); - l = left; - r = right; + if (PyJitRef_IsUnique(left)) { + ADD_OP(_BINARY_OP_ADD_FLOAT_INPLACE, 0, 0); + l = PyJitRef_Borrow(sym_new_null(ctx)); + r = right; + } + else if (PyJitRef_IsUnique(right)) { + ADD_OP(_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT, 0, 0); + l = left; + r = PyJitRef_Borrow(sym_new_null(ctx)); + } + else { + l = left; + r = right; + } + res = PyJitRef_MakeUnique(sym_new_type(ctx, &PyFloat_Type)); } op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res, l, r)) { - res = sym_new_type(ctx, &PyFloat_Type); - l = left; - r = right; + if (PyJitRef_IsUnique(left)) { + ADD_OP(_BINARY_OP_SUBTRACT_FLOAT_INPLACE, 0, 0); + l = PyJitRef_Borrow(sym_new_null(ctx)); + r = right; + } + else if (PyJitRef_IsUnique(right)) { + ADD_OP(_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, 0, 0); + l = left; + r = PyJitRef_Borrow(sym_new_null(ctx)); + } + else { + l = left; + r = right; + } + res = PyJitRef_MakeUnique(sym_new_type(ctx, &PyFloat_Type)); } op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res, l, r)) { - res = sym_new_type(ctx, &PyFloat_Type); - l = left; - r = right; + if (PyJitRef_IsUnique(left)) { + ADD_OP(_BINARY_OP_MULTIPLY_FLOAT_INPLACE, 0, 0); + l = PyJitRef_Borrow(sym_new_null(ctx)); + r = right; + } + else if (PyJitRef_IsUnique(right)) { + ADD_OP(_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT, 0, 0); + l = left; + r = PyJitRef_Borrow(sym_new_null(ctx)); + } + else { + l = left; + r = right; + } + res = PyJitRef_MakeUnique(sym_new_type(ctx, &PyFloat_Type)); } op(_BINARY_OP_ADD_UNICODE, (left, right -- res, l, r)) { @@ -430,9 +507,18 @@ dummy_func(void) { } op(_BINARY_OP_SUBSCR_DICT, (dict_st, sub_st -- res, ds, ss)) { + PyObject *sub = sym_get_const(ctx, sub_st); + if (sub != NULL) { + optimize_dict_known_hash(ctx, dependencies, this_instr, + sub, _BINARY_OP_SUBSCR_DICT_KNOWN_HASH); + } res = sym_new_not_null(ctx); ds = dict_st; ss = sub_st; + if (sym_is_not_container(sub_st) && + sym_matches_type(dict_st, &PyFrozenDict_Type)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(dict_st, sub_st, res); + } } op(_BINARY_OP_SUBSCR_LIST_SLICE, (list_st, sub_st -- res, ls, ss)) { @@ -515,7 +601,12 @@ dummy_func(void) { op(_UNARY_NEGATIVE, (value -- res, v)) { v = value; REPLACE_OPCODE_IF_EVALUATES_PURE(value, res); - if (sym_is_compact_int(value)) { + if (sym_matches_type(value, &PyFloat_Type) && PyJitRef_IsUnique(value)) { + ADD_OP(_UNARY_NEGATIVE_FLOAT_INPLACE, 0, 0); + v = PyJitRef_Borrow(sym_new_null(ctx)); + res = PyJitRef_MakeUnique(sym_new_type(ctx, &PyFloat_Type)); + } + else if (sym_is_compact_int(value)) { res = sym_new_compact_int(ctx); } else { @@ -616,6 +707,10 @@ dummy_func(void) { b = sym_new_type(ctx, &PyBool_Type); l = left; r = right; + if (sym_is_not_container(left) && + sym_matches_type(right, &PyFrozenSet_Type)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, b); + } } op(_CONTAINS_OP_DICT, (left, right -- b, l, r)) { @@ -713,6 +808,7 @@ dummy_func(void) { op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { assert(oparg > 0); + bottom = PyJitRef_RemoveUnique(bottom); top = bottom; } @@ -888,7 +984,7 @@ dummy_func(void) { if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; - if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { + if (co->co_argcount == oparg + sym_is_not_null(self_or_null)) { ADD_OP(_NOP, 0 ,0); } } @@ -1038,7 +1134,7 @@ dummy_func(void) { gen_frame = PyJitRef_WrapInvalid(new_frame); } - op(_SEND_GEN_FRAME, (receiver, v -- receiver, gen_frame)) { + op(_SEND_GEN_FRAME, (receiver, null, v -- receiver, null, gen_frame)) { _Py_UOpsAbstractFrame *new_frame = frame_new_from_symbol(ctx, receiver, NULL, 0); if (new_frame == NULL) { ctx->done = true; @@ -1173,6 +1269,47 @@ dummy_func(void) { none = sym_new_const(ctx, Py_None); } + op(_GUARD_CALLABLE_BUILTIN_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyCFunction_Type) && + (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { + int total_args = oparg; + if (sym_is_not_null(self_or_null)) { + total_args++; + } + if (total_args == 1 && PyCFunction_GET_FLAGS(callable_o) == METH_O) { + ADD_OP(_NOP, 0, 0); + } + } + else { + sym_set_type(callable, &PyCFunction_Type); + } + } + + op(_GUARD_CALLABLE_BUILTIN_FAST, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { + if (PyCFunction_GET_FLAGS(callable_o) == METH_FASTCALL) { + ADD_OP(_NOP, 0, 0); + } + } + else { + sym_set_type(callable, &PyCFunction_Type); + } + } + + op(_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { + if (PyCFunction_GET_FLAGS(callable_o) == (METH_FASTCALL | METH_KEYWORDS)) { + ADD_OP(_NOP, 0, 0); + } + } + else { + sym_set_type(callable, &PyCFunction_Type); + } + } + op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res, c, s)) { res = sym_new_not_null(ctx); c = callable; @@ -1188,7 +1325,162 @@ dummy_func(void) { } } + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && + (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { + int total_args = oparg; + if (sym_is_not_null(self_or_null)) { + total_args++; + } + PyTypeObject *self_type = NULL; + if (sym_is_not_null(self_or_null)) { + self_type = sym_get_type(self_or_null); + } + else { + self_type = sym_get_type(args[0]); + } + PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + if (total_args == 2 && + ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_O && + self_type == d_type) { + ADD_OP(_NOP, 0, 0); + } + } + else { + sym_set_type(callable, &PyMethodDescr_Type); + } + } + + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && + (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { + int total_args = oparg; + if (sym_is_not_null(self_or_null)) { + total_args++; + } + PyTypeObject *self_type = NULL; + if (sym_is_not_null(self_or_null)) { + self_type = sym_get_type(self_or_null); + } + else { + self_type = sym_get_type(args[0]); + } + PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + if (total_args != 0 && + ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == (METH_FASTCALL|METH_KEYWORDS) && + self_type == d_type) { + ADD_OP(_NOP, 0, 0); + } + } + else { + sym_set_type(callable, &PyMethodDescr_Type); + } + } + + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && + (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { + int total_args = oparg; + if (sym_is_not_null(self_or_null)) { + total_args++; + } + PyTypeObject *self_type = NULL; + if (sym_is_not_null(self_or_null)) { + self_type = sym_get_type(self_or_null); + } + else { + self_type = sym_get_type(args[0]); + } + PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + if (total_args == 1 && + ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_NOARGS && + self_type == d_type) { + ADD_OP(_NOP, 0, 0); + } + } + else { + sym_set_type(callable, &PyMethodDescr_Type); + } + } + + op(_CHECK_RECURSION_LIMIT, ( -- )) { + if (ctx->frame->is_c_recursion_checked) { + ADD_OP(_NOP, 0, 0); + } + ctx->frame->is_c_recursion_checked = true; + } + + op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res)) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) + && sym_is_not_null(self_or_null)) { + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyCFunction cfunc = method->d_method->ml_meth; + ADD_OP(_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE, oparg + 1, (uintptr_t)cfunc); + } + res = sym_new_not_null(ctx); + } + + op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) + && sym_is_not_null(self_or_null)) { + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyCFunction cfunc = method->d_method->ml_meth; + ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, oparg + 1, (uintptr_t)cfunc); + } + res = sym_new_not_null(ctx); + } + + op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- res)) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) + && sym_is_not_null(self_or_null)) { + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyCFunction cfunc = method->d_method->ml_meth; + ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_INLINE, oparg + 1, (uintptr_t)cfunc); + } + res = sym_new_not_null(ctx); + } + + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && + (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { + int total_args = oparg; + if (sym_is_not_null(self_or_null)) { + total_args++; + } + PyTypeObject *self_type = NULL; + if (sym_is_not_null(self_or_null)) { + self_type = sym_get_type(self_or_null); + } + else { + self_type = sym_get_type(args[0]); + } + PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + if (total_args != 0 && + ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_FASTCALL && + self_type == d_type) { + ADD_OP(_NOP, 0, 0); + } + } + else { + sym_set_type(callable, &PyMethodDescr_Type); + } + } + op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res, c, s, a)) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) + && sym_is_not_null(self_or_null)) { + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyCFunction cfunc = method->d_method->ml_meth; + ADD_OP(_CALL_METHOD_DESCRIPTOR_O_INLINE, oparg + 1, (uintptr_t)cfunc); + } res = sym_new_not_null(ctx); c = callable; if (sym_is_not_null(self_or_null)) { @@ -1207,6 +1499,12 @@ dummy_func(void) { v = value; } + op(_CALL_INTRINSIC_2, (value2_st, value1_st -- res, vs1, vs2)) { + res = sym_new_not_null(ctx); + vs1 = value1_st; + vs2 = value2_st; + } + op(_GUARD_IS_TRUE_POP, (flag -- )) { sym_apply_predicate_narrowing(ctx, flag, true); @@ -1311,6 +1609,7 @@ dummy_func(void) { op(_BUILD_TUPLE, (values[oparg] -- tup)) { tup = sym_new_tuple(ctx, oparg, values); + tup = PyJitRef_MakeUnique(tup); } op(_BUILD_LIST, (values[oparg] -- list)) { @@ -1338,12 +1637,32 @@ dummy_func(void) { i = iterable; } + op(_LIST_EXTEND, (list_st, unused[oparg-1], iterable_st -- list_st, unused[oparg-1], i)) { + (void)list_st; + i = iterable_st; + } + + op(_DICT_MERGE, (callable, unused, unused, dict, unused[oparg - 1], update -- callable, unused, unused, dict, unused[oparg - 1], u)) { + (void)callable; + (void)dict; + u = update; + } + op(_UNPACK_SEQUENCE_TWO_TUPLE, (seq -- val1, val0)) { + if (PyJitRef_IsUnique(seq) && sym_tuple_length(seq) == 2) { + ADD_OP(_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, oparg, 0); + } val0 = sym_tuple_getitem(ctx, seq, 0); val1 = sym_tuple_getitem(ctx, seq, 1); } op(_UNPACK_SEQUENCE_TUPLE, (seq -- values[oparg])) { + if (PyJitRef_IsUnique(seq) && sym_tuple_length(seq) == 3) { + ADD_OP(_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE, oparg, 0); + } + else if (PyJitRef_IsUnique(seq) && sym_tuple_length(seq) == oparg) { + ADD_OP(_UNPACK_SEQUENCE_UNIQUE_TUPLE, oparg, 0); + } for (int i = 0; i < oparg; i++) { values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1); } @@ -1752,6 +2071,11 @@ dummy_func(void) { n = names; } + op(_DICT_UPDATE, (dict, unused[oparg - 1], update -- dict, unused[oparg - 1], upd)) { + (void)dict; + upd = update; + } + op(_RECORD_TOS, (tos -- tos)) { sym_set_recorded_value(tos, (PyObject *)this_instr->operand0); } @@ -1779,6 +2103,12 @@ dummy_func(void) { sym_set_recorded_gen_func(nos, func); } + op(_RECORD_3OS_GEN_FUNC, (gen, nos, tos -- gen, nos, tos)) { + PyFunctionObject *func = (PyFunctionObject *)this_instr->operand0; + assert(func == NULL || PyFunction_Check(func)); + sym_set_recorded_gen_func(gen, func); + } + op(_GUARD_CODE_VERSION__PUSH_FRAME, (version/2 -- )) { PyCodeObject *co = get_current_code_object(ctx); if (co->co_version == version) { @@ -1794,18 +2124,21 @@ dummy_func(void) { } op(_GUARD_CODE_VERSION_RETURN_VALUE, (version/2 -- )) { + (void)version; if (ctx->frame->caller) { REPLACE_OP(this_instr, _NOP, 0, 0); } } op(_GUARD_CODE_VERSION_YIELD_VALUE, (version/2 -- )) { + (void)version; if (ctx->frame->caller) { REPLACE_OP(this_instr, _NOP, 0, 0); } } op(_GUARD_CODE_VERSION_RETURN_GENERATOR, (version/2 -- )) { + (void)version; if (ctx->frame->caller) { REPLACE_OP(this_instr, _NOP, 0, 0); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 40938a4411e3ec..a15b5ae1d13d3b 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -31,6 +31,7 @@ if (sym_is_null(value)) { ctx->done = true; } + assert(!PyJitRef_IsUnique(value)); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; @@ -41,6 +42,7 @@ case _LOAD_FAST: { JitOptRef value; value = GETLOCAL(oparg); + assert(!PyJitRef_IsUnique(value)); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; @@ -51,6 +53,7 @@ case _LOAD_FAST_BORROW: { JitOptRef value; value = PyJitRef_Borrow(GETLOCAL(oparg)); + assert(!PyJitRef_IsUnique(value)); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; @@ -63,6 +66,7 @@ value = GETLOCAL(oparg); JitOptRef temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; + assert(!PyJitRef_IsUnique(value)); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; @@ -102,7 +106,7 @@ JitOptRef trash; value = stack_pointer[-1]; JitOptRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; + GETLOCAL(oparg) = PyJitRef_RemoveUnique(value); trash = tmp; stack_pointer[-1] = trash; break; @@ -209,9 +213,9 @@ case _END_SEND: { JitOptRef val; val = sym_new_not_null(ctx); - CHECK_STACK_BOUNDS(-1); - stack_pointer[-2] = val; - stack_pointer += -1; + CHECK_STACK_BOUNDS(-2); + stack_pointer[-3] = val; + stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -253,7 +257,12 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } - if (sym_is_compact_int(value)) { + if (sym_matches_type(value, &PyFloat_Type) && PyJitRef_IsUnique(value)) { + ADD_OP(_UNARY_NEGATIVE_FLOAT_INPLACE, 0, 0); + v = PyJitRef_Borrow(sym_new_null(ctx)); + res = PyJitRef_MakeUnique(sym_new_type(ctx, &PyFloat_Type)); + } + else if (sym_is_compact_int(value)) { res = sym_new_compact_int(ctx); } else { @@ -273,6 +282,19 @@ break; } + case _UNARY_NEGATIVE_FLOAT_INPLACE: { + JitOptRef res; + JitOptRef v; + res = sym_new_not_null(ctx); + v = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-1] = res; + stack_pointer[0] = v; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + case _UNARY_NOT: { JitOptRef value; JitOptRef res; @@ -574,7 +596,13 @@ JitOptRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; - res = sym_new_compact_int(ctx); + if (PyJitRef_IsUnique(left)) { + REPLACE_OP(this_instr, _BINARY_OP_MULTIPLY_INT_INPLACE, 0, 0); + } + else if (PyJitRef_IsUnique(right)) { + REPLACE_OP(this_instr, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, 0, 0); + } + res = PyJitRef_MakeUnique(sym_new_compact_int(ctx)); l = left; r = right; if ( @@ -638,7 +666,13 @@ JitOptRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; - res = sym_new_compact_int(ctx); + if (PyJitRef_IsUnique(left)) { + REPLACE_OP(this_instr, _BINARY_OP_ADD_INT_INPLACE, 0, 0); + } + else if (PyJitRef_IsUnique(right)) { + REPLACE_OP(this_instr, _BINARY_OP_ADD_INT_INPLACE_RIGHT, 0, 0); + } + res = PyJitRef_MakeUnique(sym_new_compact_int(ctx)); l = left; r = right; if ( @@ -702,7 +736,13 @@ JitOptRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; - res = sym_new_compact_int(ctx); + if (PyJitRef_IsUnique(left)) { + REPLACE_OP(this_instr, _BINARY_OP_SUBTRACT_INT_INPLACE, 0, 0); + } + else if (PyJitRef_IsUnique(right)) { + REPLACE_OP(this_instr, _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, 0, 0); + } + res = PyJitRef_MakeUnique(sym_new_compact_int(ctx)); l = left; r = right; if ( @@ -758,6 +798,102 @@ break; } + case _BINARY_OP_ADD_INT_INPLACE: { + JitOptRef res; + JitOptRef l; + JitOptRef r; + res = sym_new_not_null(ctx); + l = sym_new_not_null(ctx); + r = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_SUBTRACT_INT_INPLACE: { + JitOptRef res; + JitOptRef l; + JitOptRef r; + res = sym_new_not_null(ctx); + l = sym_new_not_null(ctx); + r = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_MULTIPLY_INT_INPLACE: { + JitOptRef res; + JitOptRef l; + JitOptRef r; + res = sym_new_not_null(ctx); + l = sym_new_not_null(ctx); + r = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_ADD_INT_INPLACE_RIGHT: { + JitOptRef res; + JitOptRef l; + JitOptRef r; + res = sym_new_not_null(ctx); + l = sym_new_not_null(ctx); + r = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT: { + JitOptRef res; + JitOptRef l; + JitOptRef r; + res = sym_new_not_null(ctx); + l = sym_new_not_null(ctx); + r = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT: { + JitOptRef res; + JitOptRef l; + JitOptRef r; + res = sym_new_not_null(ctx); + l = sym_new_not_null(ctx); + r = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + case _GUARD_NOS_FLOAT: { JitOptRef left; left = stack_pointer[-2]; @@ -786,9 +922,21 @@ JitOptRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; - res = sym_new_type(ctx, &PyFloat_Type); - l = left; - r = right; + if (PyJitRef_IsUnique(left)) { + ADD_OP(_BINARY_OP_MULTIPLY_FLOAT_INPLACE, 0, 0); + l = PyJitRef_Borrow(sym_new_null(ctx)); + r = right; + } + else if (PyJitRef_IsUnique(right)) { + ADD_OP(_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT, 0, 0); + l = left; + r = PyJitRef_Borrow(sym_new_null(ctx)); + } + else { + l = left; + r = right; + } + res = PyJitRef_MakeUnique(sym_new_type(ctx, &PyFloat_Type)); CHECK_STACK_BOUNDS(1); stack_pointer[-2] = res; stack_pointer[-1] = l; @@ -806,9 +954,21 @@ JitOptRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; - res = sym_new_type(ctx, &PyFloat_Type); - l = left; - r = right; + if (PyJitRef_IsUnique(left)) { + ADD_OP(_BINARY_OP_ADD_FLOAT_INPLACE, 0, 0); + l = PyJitRef_Borrow(sym_new_null(ctx)); + r = right; + } + else if (PyJitRef_IsUnique(right)) { + ADD_OP(_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT, 0, 0); + l = left; + r = PyJitRef_Borrow(sym_new_null(ctx)); + } + else { + l = left; + r = right; + } + res = PyJitRef_MakeUnique(sym_new_type(ctx, &PyFloat_Type)); CHECK_STACK_BOUNDS(1); stack_pointer[-2] = res; stack_pointer[-1] = l; @@ -826,9 +986,117 @@ JitOptRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; - res = sym_new_type(ctx, &PyFloat_Type); - l = left; - r = right; + if (PyJitRef_IsUnique(left)) { + ADD_OP(_BINARY_OP_SUBTRACT_FLOAT_INPLACE, 0, 0); + l = PyJitRef_Borrow(sym_new_null(ctx)); + r = right; + } + else if (PyJitRef_IsUnique(right)) { + ADD_OP(_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, 0, 0); + l = left; + r = PyJitRef_Borrow(sym_new_null(ctx)); + } + else { + l = left; + r = right; + } + res = PyJitRef_MakeUnique(sym_new_type(ctx, &PyFloat_Type)); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_ADD_FLOAT_INPLACE: { + JitOptRef res; + JitOptRef l; + JitOptRef r; + res = sym_new_not_null(ctx); + l = sym_new_not_null(ctx); + r = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE: { + JitOptRef res; + JitOptRef l; + JitOptRef r; + res = sym_new_not_null(ctx); + l = sym_new_not_null(ctx); + r = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE: { + JitOptRef res; + JitOptRef l; + JitOptRef r; + res = sym_new_not_null(ctx); + l = sym_new_not_null(ctx); + r = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT: { + JitOptRef res; + JitOptRef l; + JitOptRef r; + res = sym_new_not_null(ctx); + l = sym_new_not_null(ctx); + r = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT: { + JitOptRef res; + JitOptRef l; + JitOptRef r; + res = sym_new_not_null(ctx); + l = sym_new_not_null(ctx); + r = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT: { + JitOptRef res; + JitOptRef l; + JitOptRef r; + res = sym_new_not_null(ctx); + l = sym_new_not_null(ctx); + r = sym_new_not_null(ctx); CHECK_STACK_BOUNDS(1); stack_pointer[-2] = res; stack_pointer[-1] = l; @@ -1162,6 +1430,22 @@ break; } + case _BINARY_OP_SUBSCR_DICT_KNOWN_HASH: { + JitOptRef res; + JitOptRef ds; + JitOptRef ss; + res = sym_new_not_null(ctx); + ds = sym_new_not_null(ctx); + ss = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = ds; + stack_pointer[0] = ss; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + case _BINARY_OP_SUBSCR_DICT: { JitOptRef sub_st; JitOptRef dict_st; @@ -1170,9 +1454,63 @@ JitOptRef ss; sub_st = stack_pointer[-1]; dict_st = stack_pointer[-2]; + PyObject *sub = sym_get_const(ctx, sub_st); + if (sub != NULL) { + optimize_dict_known_hash(ctx, dependencies, this_instr, + sub, _BINARY_OP_SUBSCR_DICT_KNOWN_HASH); + } res = sym_new_not_null(ctx); ds = dict_st; ss = sub_st; + if (sym_is_not_container(sub_st) && + sym_matches_type(dict_st, &PyFrozenDict_Type)) { + if ( + sym_is_safe_const(ctx, dict_st) && + sym_is_safe_const(ctx, sub_st) + ) { + JitOptRef dict_st_sym = dict_st; + JitOptRef sub_st_sym = sub_st; + _PyStackRef dict_st = sym_get_const_as_stackref(ctx, dict_st_sym); + _PyStackRef sub_st = sym_get_const_as_stackref(ctx, sub_st_sym); + _PyStackRef res_stackref; + _PyStackRef ds_stackref; + _PyStackRef ss_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyAnyDict_CheckExact(dict)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o; + int rc = PyDict_GetItemRef(dict, sub, &res_o); + if (rc == 0) { + _PyErr_SetKeyError(sub); + } + if (rc <= 0) { + JUMP_TO_LABEL(error); + } + res_stackref = PyStackRef_FromPyObjectSteal(res_o); + ds_stackref = dict_st; + ss_stackref = sub_st; + /* End of uop copied from bytecodes for constant evaluation */ + (void)ds_stackref; + (void)ss_stackref; + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = ds; + stack_pointer[0] = ss; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + } CHECK_STACK_BOUNDS(1); stack_pointer[-2] = res; stack_pointer[-1] = ds; @@ -1269,11 +1607,18 @@ } case _STORE_SUBSCR_DICT: { + JitOptRef sub; JitOptRef dict_st; JitOptRef value; JitOptRef st; + sub = stack_pointer[-1]; dict_st = stack_pointer[-2]; value = stack_pointer[-3]; + PyObject *sub_o = sym_get_const(ctx, sub); + if (sub_o != NULL) { + optimize_dict_known_hash(ctx, dependencies, this_instr, + sub_o, _STORE_SUBSCR_DICT_KNOWN_HASH); + } (void)value; st = dict_st; CHECK_STACK_BOUNDS(-2); @@ -1283,6 +1628,16 @@ break; } + case _STORE_SUBSCR_DICT_KNOWN_HASH: { + JitOptRef st; + st = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-2); + stack_pointer[-3] = st; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + case _DELETE_SUBSCR: { CHECK_STACK_BOUNDS(-2); stack_pointer += -2; @@ -1306,11 +1661,21 @@ } case _CALL_INTRINSIC_2: { + JitOptRef value1_st; + JitOptRef value2_st; JitOptRef res; + JitOptRef vs1; + JitOptRef vs2; + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; res = sym_new_not_null(ctx); - CHECK_STACK_BOUNDS(-1); + vs1 = value1_st; + vs2 = value2_st; + CHECK_STACK_BOUNDS(1); stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[-1] = vs1; + stack_pointer[0] = vs2; + stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1321,7 +1686,7 @@ if (sym_is_immortal(PyJitRef_Unwrap(value))) { ADD_OP(_NOP, 0, 0); } - value = PyJitRef_StripReferenceInfo(value); + value = PyJitRef_StripBorrowInfo(value); stack_pointer[-1] = value; break; } @@ -1385,7 +1750,7 @@ JitOptRef receiver; JitOptRef gen_frame; v = stack_pointer[-1]; - receiver = stack_pointer[-2]; + receiver = stack_pointer[-3]; _Py_UOpsAbstractFrame *new_frame = frame_new_from_symbol(ctx, receiver, NULL, 0); if (new_frame == NULL) { ctx->done = true; @@ -1487,6 +1852,9 @@ JitOptRef val1; JitOptRef val0; seq = stack_pointer[-1]; + if (PyJitRef_IsUnique(seq) && sym_tuple_length(seq) == 2) { + ADD_OP(_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, oparg, 0); + } val0 = sym_tuple_getitem(ctx, seq, 0); val1 = sym_tuple_getitem(ctx, seq, 1); CHECK_STACK_BOUNDS(1); @@ -1497,11 +1865,46 @@ break; } + case _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE: { + JitOptRef val1; + JitOptRef val0; + val1 = sym_new_not_null(ctx); + val0 = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE: { + JitOptRef val2; + JitOptRef val1; + JitOptRef val0; + val2 = sym_new_not_null(ctx); + val1 = sym_new_not_null(ctx); + val0 = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(2); + stack_pointer[-1] = val2; + stack_pointer[0] = val1; + stack_pointer[1] = val0; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + case _UNPACK_SEQUENCE_TUPLE: { JitOptRef seq; JitOptRef *values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; + if (PyJitRef_IsUnique(seq) && sym_tuple_length(seq) == 3) { + ADD_OP(_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE, oparg, 0); + } + else if (PyJitRef_IsUnique(seq) && sym_tuple_length(seq) == oparg) { + ADD_OP(_UNPACK_SEQUENCE_UNIQUE_TUPLE, oparg, 0); + } for (int i = 0; i < oparg; i++) { values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1); } @@ -1511,6 +1914,18 @@ break; } + case _UNPACK_SEQUENCE_UNIQUE_TUPLE: { + JitOptRef *values; + values = &stack_pointer[-1]; + for (int _i = oparg; --_i >= 0;) { + values[_i] = sym_new_not_null(ctx); + } + CHECK_STACK_BOUNDS(-1 + oparg); + stack_pointer += -1 + oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + case _UNPACK_SEQUENCE_LIST: { JitOptRef *values; values = &stack_pointer[-1]; @@ -1763,6 +2178,15 @@ } case _COPY_FREE_VARS: { + PyCodeObject *co = get_current_code_object(ctx); + if (co == NULL) { + ctx->done = true; + break; + } + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + ctx->frame->locals[offset + i] = sym_new_not_null(ctx); + } break; } @@ -1801,6 +2225,7 @@ JitOptRef tup; values = &stack_pointer[-oparg]; tup = sym_new_tuple(ctx, oparg, values); + tup = PyJitRef_MakeUnique(tup); CHECK_STACK_BOUNDS(1 - oparg); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; @@ -1819,9 +2244,14 @@ } case _LIST_EXTEND: { - CHECK_STACK_BOUNDS(-1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JitOptRef iterable_st; + JitOptRef list_st; + JitOptRef i; + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + (void)list_st; + i = iterable_st; + stack_pointer[-1] = i; break; } @@ -1862,16 +2292,29 @@ } case _DICT_UPDATE: { - CHECK_STACK_BOUNDS(-1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JitOptRef update; + JitOptRef dict; + JitOptRef upd; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + (void)dict; + upd = update; + stack_pointer[-1] = upd; break; } case _DICT_MERGE: { - CHECK_STACK_BOUNDS(-1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JitOptRef update; + JitOptRef dict; + JitOptRef callable; + JitOptRef u; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + (void)callable; + (void)dict; + u = update; + stack_pointer[-1] = u; break; } @@ -2551,6 +2994,51 @@ b = sym_new_type(ctx, &PyBool_Type); l = left; r = right; + if (sym_is_not_container(left) && + sym_matches_type(right, &PyFrozenSet_Type)) { + if ( + sym_is_safe_const(ctx, left) && + sym_is_safe_const(ctx, right) + ) { + JitOptRef left_sym = left; + JitOptRef right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef b_stackref; + _PyStackRef l_stackref; + _PyStackRef r_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyAnySet_CheckExact(right_o)); + STAT_INC(CONTAINS_OP, hit); + int res = _PySet_Contains((PySetObject *)right_o, left_o); + if (res < 0) { + JUMP_TO_LABEL(error); + } + b_stackref = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + l_stackref = left; + r_stackref = right; + /* End of uop copied from bytecodes for constant evaluation */ + (void)l_stackref; + (void)r_stackref; + b = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(b_stackref)); + if (sym_is_const(ctx, b)) { + PyObject *result = sym_get_const(ctx, b); + if (_Py_IsImmortal(result)) { + // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = b; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + } CHECK_STACK_BOUNDS(1); stack_pointer[-2] = b; stack_pointer[-1] = l; @@ -2738,13 +3226,6 @@ break; } - case _GET_YIELD_FROM_ITER: { - JitOptRef iter; - iter = sym_new_not_null(ctx); - stack_pointer[-1] = iter; - break; - } - /* _FOR_ITER is not a viable micro-op for tier 2 */ case _FOR_ITER_TIER_TWO: { @@ -3130,7 +3611,7 @@ if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; - if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { + if (co->co_argcount == oparg + sym_is_not_null(self_or_null)) { ADD_OP(_NOP, 0 ,0); } } @@ -3394,6 +3875,28 @@ break; } + case _GUARD_CALLABLE_BUILTIN_O: { + JitOptRef self_or_null; + JitOptRef callable; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyCFunction_Type) && + (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { + int total_args = oparg; + if (sym_is_not_null(self_or_null)) { + total_args++; + } + if (total_args == 1 && PyCFunction_GET_FLAGS(callable_o) == METH_O) { + ADD_OP(_NOP, 0, 0); + } + } + else { + sym_set_type(callable, &PyCFunction_Type); + } + break; + } + case _CALL_BUILTIN_O: { JitOptRef *args; JitOptRef self_or_null; @@ -3425,6 +3928,21 @@ break; } + case _GUARD_CALLABLE_BUILTIN_FAST: { + JitOptRef callable; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { + if (PyCFunction_GET_FLAGS(callable_o) == METH_FASTCALL) { + ADD_OP(_NOP, 0, 0); + } + } + else { + sym_set_type(callable, &PyCFunction_Type); + } + break; + } + case _CALL_BUILTIN_FAST: { JitOptRef res; res = sym_new_not_null(ctx); @@ -3435,6 +3953,21 @@ break; } + case _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS: { + JitOptRef callable; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { + if (PyCFunction_GET_FLAGS(callable_o) == (METH_FASTCALL | METH_KEYWORDS)) { + ADD_OP(_NOP, 0, 0); + } + } + else { + sym_set_type(callable, &PyCFunction_Type); + } + break; + } + case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { JitOptRef res; res = sym_new_not_null(ctx); @@ -3578,6 +4111,40 @@ break; } + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_O: { + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef callable; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && + (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { + int total_args = oparg; + if (sym_is_not_null(self_or_null)) { + total_args++; + } + PyTypeObject *self_type = NULL; + if (sym_is_not_null(self_or_null)) { + self_type = sym_get_type(self_or_null); + } + else { + self_type = sym_get_type(args[0]); + } + PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + if (total_args == 2 && + ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_O && + self_type == d_type) { + ADD_OP(_NOP, 0, 0); + } + } + else { + sym_set_type(callable, &PyMethodDescr_Type); + } + break; + } + case _CALL_METHOD_DESCRIPTOR_O: { JitOptRef *args; JitOptRef self_or_null; @@ -3589,6 +4156,13 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) + && sym_is_not_null(self_or_null)) { + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyCFunction cfunc = method->d_method->ml_meth; + ADD_OP(_CALL_METHOD_DESCRIPTOR_O_INLINE, oparg + 1, (uintptr_t)cfunc); + } res = sym_new_not_null(ctx); c = callable; if (sym_is_not_null(self_or_null)) { @@ -3610,8 +4184,80 @@ break; } + case _CHECK_RECURSION_LIMIT: { + if (ctx->frame->is_c_recursion_checked) { + ADD_OP(_NOP, 0, 0); + } + ctx->frame->is_c_recursion_checked = true; + break; + } + + case _CALL_METHOD_DESCRIPTOR_O_INLINE: { + JitOptRef res; + JitOptRef c; + JitOptRef s; + JitOptRef a; + res = sym_new_not_null(ctx); + c = sym_new_not_null(ctx); + s = sym_new_not_null(ctx); + a = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(3 - oparg); + stack_pointer[-1 - oparg] = res; + stack_pointer[-oparg] = c; + stack_pointer[1 - oparg] = s; + stack_pointer[2 - oparg] = a; + stack_pointer += 3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef callable; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && + (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { + int total_args = oparg; + if (sym_is_not_null(self_or_null)) { + total_args++; + } + PyTypeObject *self_type = NULL; + if (sym_is_not_null(self_or_null)) { + self_type = sym_get_type(self_or_null); + } + else { + self_type = sym_get_type(args[0]); + } + PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + if (total_args != 0 && + ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == (METH_FASTCALL|METH_KEYWORDS) && + self_type == d_type) { + ADD_OP(_NOP, 0, 0); + } + } + else { + sym_set_type(callable, &PyMethodDescr_Type); + } + break; + } + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { + JitOptRef self_or_null; + JitOptRef callable; JitOptRef res; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) + && sym_is_not_null(self_or_null)) { + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyCFunction cfunc = method->d_method->ml_meth; + ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, oparg + 1, (uintptr_t)cfunc); + } res = sym_new_not_null(ctx); CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; @@ -3620,8 +4266,63 @@ break; } + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE: { + JitOptRef res; + res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-oparg); + stack_pointer[-1 - oparg] = res; + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS: { + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef callable; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && + (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { + int total_args = oparg; + if (sym_is_not_null(self_or_null)) { + total_args++; + } + PyTypeObject *self_type = NULL; + if (sym_is_not_null(self_or_null)) { + self_type = sym_get_type(self_or_null); + } + else { + self_type = sym_get_type(args[0]); + } + PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + if (total_args == 1 && + ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_NOARGS && + self_type == d_type) { + ADD_OP(_NOP, 0, 0); + } + } + else { + sym_set_type(callable, &PyMethodDescr_Type); + } + break; + } + case _CALL_METHOD_DESCRIPTOR_NOARGS: { + JitOptRef self_or_null; + JitOptRef callable; JitOptRef res; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) + && sym_is_not_null(self_or_null)) { + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyCFunction cfunc = method->d_method->ml_meth; + ADD_OP(_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE, oparg + 1, (uintptr_t)cfunc); + } res = sym_new_not_null(ctx); CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; @@ -3630,8 +4331,63 @@ break; } + case _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE: { + JitOptRef res; + res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-oparg); + stack_pointer[-1 - oparg] = res; + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST: { + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef callable; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && + (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { + int total_args = oparg; + if (sym_is_not_null(self_or_null)) { + total_args++; + } + PyTypeObject *self_type = NULL; + if (sym_is_not_null(self_or_null)) { + self_type = sym_get_type(self_or_null); + } + else { + self_type = sym_get_type(args[0]); + } + PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + if (total_args != 0 && + ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_FASTCALL && + self_type == d_type) { + ADD_OP(_NOP, 0, 0); + } + } + else { + sym_set_type(callable, &PyMethodDescr_Type); + } + break; + } + case _CALL_METHOD_DESCRIPTOR_FAST: { + JitOptRef self_or_null; + JitOptRef callable; JitOptRef res; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) + && sym_is_not_null(self_or_null)) { + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyCFunction cfunc = method->d_method->ml_meth; + ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_INLINE, oparg + 1, (uintptr_t)cfunc); + } res = sym_new_not_null(ctx); CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; @@ -3640,6 +4396,16 @@ break; } + case _CALL_METHOD_DESCRIPTOR_FAST_INLINE: { + JitOptRef res; + res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-oparg); + stack_pointer[-1 - oparg] = res; + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + /* _MONITOR_CALL_KW is not a viable micro-op for tier 2 */ case _MAYBE_EXPAND_METHOD_KW: { @@ -3800,8 +4566,10 @@ JitOptRef top; bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); + bottom = PyJitRef_RemoveUnique(bottom); top = bottom; CHECK_STACK_BOUNDS(1); + stack_pointer[-1 - (oparg-1)] = bottom; stack_pointer[0] = top; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -4324,6 +5092,7 @@ case _GUARD_CODE_VERSION_YIELD_VALUE: { uint32_t version = (uint32_t)this_instr->operand0; + (void)version; if (ctx->frame->caller) { REPLACE_OP(this_instr, _NOP, 0, 0); } @@ -4332,6 +5101,7 @@ case _GUARD_CODE_VERSION_RETURN_VALUE: { uint32_t version = (uint32_t)this_instr->operand0; + (void)version; if (ctx->frame->caller) { REPLACE_OP(this_instr, _NOP, 0, 0); } @@ -4340,6 +5110,7 @@ case _GUARD_CODE_VERSION_RETURN_GENERATOR: { uint32_t version = (uint32_t)this_instr->operand0; + (void)version; if (ctx->frame->caller) { REPLACE_OP(this_instr, _NOP, 0, 0); } @@ -4420,6 +5191,15 @@ break; } + case _RECORD_3OS_GEN_FUNC: { + JitOptRef gen; + gen = stack_pointer[-3]; + PyFunctionObject *func = (PyFunctionObject *)this_instr->operand0; + assert(func == NULL || PyFunction_Check(func)); + sym_set_recorded_gen_func(gen, func); + break; + } + case _RECORD_4OS: { JitOptRef value; value = stack_pointer[-4]; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index c8697b32ab2dd4..6230b8948697e2 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -282,6 +282,22 @@ _Py_uop_sym_is_safe_const(JitOptContext *ctx, JitOptRef sym) return (typ == &PyUnicode_Type) || (typ == &PyFloat_Type) || (typ == &_PyNone_Type) || + (typ == &PyBool_Type) || + (typ == &PyFrozenDict_Type) || + (typ == &PyFrozenSet_Type); +} + +bool +_Py_uop_sym_is_not_container(JitOptRef sym) +{ + PyTypeObject *typ = _Py_uop_sym_get_type(sym); + if (typ == NULL) { + return false; + } + return (typ == &PyLong_Type) || + (typ == &PyFloat_Type) || + (typ == &PyUnicode_Type) || + (typ == &_PyNone_Type) || (typ == &PyBool_Type); } @@ -522,7 +538,7 @@ _Py_uop_sym_set_func_version(JitOptContext *ctx, JitOptRef ref, uint32_t version case JIT_SYM_PREDICATE_TAG: case JIT_SYM_TRUTHINESS_TAG: sym_set_bottom(ctx, sym); - return true; + return false; case JIT_SYM_RECORDED_VALUE_TAG: { PyObject *val = sym->recorded_value.value; if (Py_TYPE(val) != &PyFunction_Type || @@ -1513,6 +1529,7 @@ _Py_uop_frame_new( frame->globals_watched = false; frame->func = NULL; frame->caller = false; + frame->is_c_recursion_checked = false; if (ctx->locals.used > ctx->locals.end || ctx->stack.used > ctx->stack.end) { ctx->done = true; ctx->out_of_space = true; @@ -1521,7 +1538,7 @@ _Py_uop_frame_new( // Initialize with the initial state of all local variables for (int i = 0; i < arg_len; i++) { - frame->locals[i] = args[i]; + frame->locals[i] = PyJitRef_RemoveUnique(args[i]); } // If the args are known, then it's safe to just initialize @@ -1684,6 +1701,9 @@ static JitOptSymbol * make_bottom(JitOptContext *ctx) { JitOptSymbol *sym = sym_new(ctx); + if (sym == NULL) { + return out_of_space(ctx); + } sym->tag = JIT_SYM_BOTTOM_TAG; return sym; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 68052a91d6a1f1..b3832ad4c87f88 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -41,6 +41,10 @@ #include "pycore_jit.h" // _PyJIT_Fini() #endif +#if defined(PYMALLOC_USE_HUGEPAGES) && defined(MS_WINDOWS) +#include +#endif + #include "opcode.h" #include // setlocale() @@ -486,6 +490,41 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime, return _PyStatus_OK(); } +#if defined(PYMALLOC_USE_HUGEPAGES) && defined(MS_WINDOWS) +static PyStatus +get_huge_pages_privilege(void) +{ + HANDLE hToken; + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hToken)) + { + return _PyStatus_ERR("failed to open process token"); + } + TOKEN_PRIVILEGES tp; + if (!LookupPrivilegeValue(NULL, SE_LOCK_MEMORY_NAME, &tp.Privileges[0].Luid)) + { + CloseHandle(hToken); + return _PyStatus_ERR("failed to lookup SeLockMemoryPrivilege for huge pages"); + } + tp.PrivilegeCount = 1; + tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; + // AdjustTokenPrivileges can return with nonzero status (i.e. success) + // but without having all privileges adjusted (ERROR_NOT_ALL_ASSIGNED). + BOOL status = AdjustTokenPrivileges(hToken, FALSE, &tp, sizeof(TOKEN_PRIVILEGES), NULL, NULL); + DWORD error = GetLastError(); + if (!status || (error != ERROR_SUCCESS)) + { + CloseHandle(hToken); + return _PyStatus_ERR( + "SeLockMemoryPrivilege not held; " + "grant it via Local Security Policy or disable PYTHON_PYMALLOC_HUGEPAGES"); + } + if (!CloseHandle(hToken)) + { + return _PyStatus_ERR("failed to close process token handle"); + } + return _PyStatus_OK(); +} +#endif static PyStatus pycore_init_runtime(_PyRuntimeState *runtime, @@ -500,6 +539,15 @@ pycore_init_runtime(_PyRuntimeState *runtime, return status; } +#if defined(PYMALLOC_USE_HUGEPAGES) && defined(MS_WINDOWS) + if (runtime->allocators.use_hugepages) { + status = get_huge_pages_privilege(); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } +#endif + /* Py_Finalize leaves _Py_Finalizing set in order to help daemon * threads behave a little more gracefully at interpreter shutdown. * We clobber it here so the new interpreter can start with a clean @@ -3213,9 +3261,9 @@ _Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, /* display the current Python stack */ #ifndef Py_GIL_DISABLED - _Py_DumpTracebackThreads(fd, interp, tstate); + PyUnstable_DumpTracebackThreads(fd, interp, tstate); #else - _Py_DumpTraceback(fd, tstate); + PyUnstable_DumpTraceback(fd, tstate); #endif } diff --git a/Python/pystrhex.c b/Python/pystrhex.c index 698e7f26fbaae7..14d5719313afd2 100644 --- a/Python/pystrhex.c +++ b/Python/pystrhex.c @@ -111,9 +111,10 @@ _Py_hexlify_simd(const unsigned char *src, Py_UCS1 *dst, Py_ssize_t len) #endif /* HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR */ -static PyObject *_Py_strhex_impl(const char* argbuf, const Py_ssize_t arglen, - PyObject* sep, int bytes_per_sep_group, - const int return_bytes) +static PyObject * +_Py_strhex_impl(const char* argbuf, Py_ssize_t arglen, + PyObject* sep, Py_ssize_t bytes_per_sep_group, + int return_bytes) { assert(arglen >= 0); @@ -149,7 +150,7 @@ static PyObject *_Py_strhex_impl(const char* argbuf, const Py_ssize_t arglen, else { bytes_per_sep_group = 0; } - unsigned int abs_bytes_per_sep = _Py_ABS_CAST(unsigned int, bytes_per_sep_group); + size_t abs_bytes_per_sep = _Py_ABS_CAST(size_t, bytes_per_sep_group); Py_ssize_t resultlen = 0; if (bytes_per_sep_group && arglen > 0) { /* How many sep characters we'll be inserting. */ @@ -203,7 +204,7 @@ static PyObject *_Py_strhex_impl(const char* argbuf, const Py_ssize_t arglen, /* The number of complete chunk+sep periods */ Py_ssize_t chunks = (arglen - 1) / abs_bytes_per_sep; Py_ssize_t chunk; - unsigned int k; + size_t k; if (bytes_per_sep_group < 0) { i = j = 0; @@ -251,30 +252,30 @@ static PyObject *_Py_strhex_impl(const char* argbuf, const Py_ssize_t arglen, return retval; } -PyObject * _Py_strhex(const char* argbuf, const Py_ssize_t arglen) +PyObject * _Py_strhex(const char* argbuf, Py_ssize_t arglen) { return _Py_strhex_impl(argbuf, arglen, NULL, 0, 0); } /* Same as above but returns a bytes() instead of str() to avoid the * need to decode the str() when bytes are needed. */ -PyObject* _Py_strhex_bytes(const char* argbuf, const Py_ssize_t arglen) +PyObject* _Py_strhex_bytes(const char* argbuf, Py_ssize_t arglen) { return _Py_strhex_impl(argbuf, arglen, NULL, 0, 1); } /* These variants include support for a separator between every N bytes: */ -PyObject* _Py_strhex_with_sep(const char* argbuf, const Py_ssize_t arglen, - PyObject* sep, const int bytes_per_group) +PyObject* _Py_strhex_with_sep(const char* argbuf, Py_ssize_t arglen, + PyObject* sep, Py_ssize_t bytes_per_group) { return _Py_strhex_impl(argbuf, arglen, sep, bytes_per_group, 0); } /* Same as above but returns a bytes() instead of str() to avoid the * need to decode the str() when bytes are needed. */ -PyObject* _Py_strhex_bytes_with_sep(const char* argbuf, const Py_ssize_t arglen, - PyObject* sep, const int bytes_per_group) +PyObject* _Py_strhex_bytes_with_sep(const char* argbuf, Py_ssize_t arglen, + PyObject* sep, Py_ssize_t bytes_per_group) { return _Py_strhex_impl(argbuf, arglen, sep, bytes_per_group, 1); } diff --git a/Python/pythonrun.c b/Python/pythonrun.c index a21f494dc69d82..971ab064777a41 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -567,6 +567,7 @@ _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompil PyObject* the_name = PyUnicode_FromString(name); if (!the_name) { PyErr_Print(); + Py_DECREF(main_module); return -1; } res = _PyRun_StringFlagsWithName(command, the_name, Py_file_input, dict, dict, flags, 0); @@ -1381,11 +1382,11 @@ get_interactive_filename(PyObject *filename, Py_ssize_t count) if (middle == NULL) { return NULL; } - result = PyUnicode_FromFormat("<%U-%d>", middle, count); + result = PyUnicode_FromFormat("<%U-%zd>", middle, count); Py_DECREF(middle); } else { result = PyUnicode_FromFormat( - "%U-%d", filename, count); + "%U-%zd", filename, count); } return result; diff --git a/Python/record_functions.c.h b/Python/record_functions.c.h index 64cafcb326e111..958cdab34c8ff4 100644 --- a/Python/record_functions.c.h +++ b/Python/record_functions.c.h @@ -41,6 +41,20 @@ void _PyOpcode_RecordFunction_NOS_GEN_FUNC(_PyInterpreterFrame *frame, _PyStackR } } +void _PyOpcode_RecordFunction_3OS_GEN_FUNC(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int oparg, PyObject **recorded_value) { + _PyStackRef gen; + gen = stack_pointer[-3]; + PyObject *obj = PyStackRef_AsPyObjectBorrow(gen); + if (PyGen_Check(obj)) { + PyGenObject *gen_obj = (PyGenObject *)obj; + _PyStackRef func = gen_obj->gi_iframe.f_funcobj; + if (!PyStackRef_IsNull(func)) { + *recorded_value = (PyObject *)PyStackRef_AsPyObjectBorrow(func); + Py_INCREF(*recorded_value); + } + } +} + void _PyOpcode_RecordFunction_4OS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int oparg, PyObject **recorded_value) { _PyStackRef value; value = stack_pointer[-4]; @@ -73,14 +87,15 @@ void _PyOpcode_RecordFunction_CODE(_PyInterpreterFrame *frame, _PyStackRef *stac #define _RECORD_TOS_TYPE_INDEX 1 #define _RECORD_NOS_INDEX 2 -#define _RECORD_NOS_GEN_FUNC_INDEX 3 -#define _RECORD_CALLABLE_INDEX 4 -#define _RECORD_BOUND_METHOD_INDEX 5 -#define _RECORD_4OS_INDEX 6 +#define _RECORD_3OS_GEN_FUNC_INDEX 3 +#define _RECORD_NOS_GEN_FUNC_INDEX 4 +#define _RECORD_CALLABLE_INDEX 5 +#define _RECORD_BOUND_METHOD_INDEX 6 +#define _RECORD_4OS_INDEX 7 const uint8_t _PyOpcode_RecordFunctionIndices[256] = { [TO_BOOL_ALWAYS_TRUE] = _RECORD_TOS_TYPE_INDEX, [BINARY_OP_SUBSCR_GETITEM] = _RECORD_NOS_INDEX, - [SEND_GEN] = _RECORD_NOS_GEN_FUNC_INDEX, + [SEND_GEN] = _RECORD_3OS_GEN_FUNC_INDEX, [LOAD_ATTR_INSTANCE_VALUE] = _RECORD_TOS_TYPE_INDEX, [LOAD_ATTR_WITH_HINT] = _RECORD_TOS_TYPE_INDEX, [LOAD_ATTR_SLOT] = _RECORD_TOS_TYPE_INDEX, @@ -110,10 +125,11 @@ const uint8_t _PyOpcode_RecordFunctionIndices[256] = { [CALL_EX_PY] = _RECORD_4OS_INDEX, }; -const _Py_RecordFuncPtr _PyOpcode_RecordFunctions[7] = { +const _Py_RecordFuncPtr _PyOpcode_RecordFunctions[8] = { [0] = NULL, [_RECORD_TOS_TYPE_INDEX] = _PyOpcode_RecordFunction_TOS_TYPE, [_RECORD_NOS_INDEX] = _PyOpcode_RecordFunction_NOS, + [_RECORD_3OS_GEN_FUNC_INDEX] = _PyOpcode_RecordFunction_3OS_GEN_FUNC, [_RECORD_NOS_GEN_FUNC_INDEX] = _PyOpcode_RecordFunction_NOS_GEN_FUNC, [_RECORD_CALLABLE_INDEX] = _PyOpcode_RecordFunction_CALLABLE, [_RECORD_BOUND_METHOD_INDEX] = _PyOpcode_RecordFunction_BOUND_METHOD, diff --git a/Python/specialize.c b/Python/specialize.c index 4ef8b27795650c..09ec25767a4c3f 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -41,9 +41,24 @@ do { \ # define SPECIALIZATION_FAIL(opcode, kind) ((void)0) #endif // Py_STATS +static void +fixup_getiter(_Py_CODEUNIT *instruction, int flags) +{ + // Compiler can't know if types.coroutine() will be called, + // so fix up here + if (instruction->op.arg) { + if (flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE)) { + instruction->op.arg = GET_ITER_YIELD_FROM_NO_CHECK; + } + else { + instruction->op.arg = GET_ITER_YIELD_FROM_CORO_CHECK; + } + } +} + // Initialize warmup counters and optimize instructions. This cannot fail. void -_PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, int enable_counters) +_PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, int enable_counters, int flags) { #if ENABLE_SPECIALIZATION _Py_BackoffCounter jump_counter, adaptive_counter, resume_counter; @@ -66,6 +81,9 @@ _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, int enable_counters opcode = instructions[i].op.code; int caches = _PyOpcode_Caches[opcode]; oparg = (oparg << 8) | instructions[i].op.arg; + if (opcode == GET_ITER) { + fixup_getiter(&instructions[i], flags); + } if (caches) { // The initial value depends on the opcode switch (opcode) { @@ -91,6 +109,13 @@ _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, int enable_counters oparg = 0; } } + #else + for (Py_ssize_t i = 0; i < size-1; i++) { + if (instructions[i].op.code == GET_ITER) { + fixup_getiter(&instructions[i], flags); + } + i += _PyOpcode_Caches[opcode]; + } #endif /* ENABLE_SPECIALIZATION */ } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 646b8a1c3c3a84..ce9c03bda7bd57 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3878,20 +3878,38 @@ static PyStructSequence_Desc emscripten_info_desc = { EM_JS(char *, _Py_emscripten_runtime, (void), { var info; - if (typeof navigator == 'object') { + if (typeof process === "object") { + if (process.versions?.bun) { + info = `bun v${process.versions.bun}`; + } else if (process.versions?.deno) { + info = `deno v${process.versions.deno}`; + } else { + // As far as I can tell, every JavaScript runtime puts "node" in + // process.release.name. Pyodide once checked for + // + // process.release.name === "node" + // + // and this is apparently part of the reason other runtimes started + // lying about it. Similar to the situation with userAgent. + // + // But just in case some other JS runtime decides to tell us what it + // is, we'll pick it up. + const name = process.release?.name ?? "node"; + info = `${name} ${process.version}`; + } + // Include v8 version if we know it + if (process.versions?.v8) { + info += ` (v8 ${process.versions.v8})`; + } + } else if (typeof navigator === "object") { info = navigator.userAgent; - } else if (typeof process == 'object') { - info = "Node.js ".concat(process.version); } else { info = "UNKNOWN"; } - var len = lengthBytesUTF8(info) + 1; - var res = _malloc(len); - if (res) stringToUTF8(info, res, len); #if __wasm64__ - return BigInt(res); + return BigInt(stringToNewUTF8(info)); #else - return res; + return stringToNewUTF8(info); #endif }); diff --git a/Python/traceback.c b/Python/traceback.c index 1e8c9c879f9aac..efdbb0e1af4c9e 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -1167,10 +1167,11 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) The caller is responsible to call PyErr_CheckSignals() to call Python signal handlers if signals were received. */ -void -_Py_DumpTraceback(int fd, PyThreadState *tstate) +const char* +PyUnstable_DumpTraceback(int fd, PyThreadState *tstate) { dump_traceback(fd, tstate, 1); + return NULL; } #if defined(HAVE_PTHREAD_GETNAME_NP) || defined(HAVE_PTHREAD_GET_NAME_NP) @@ -1257,6 +1258,14 @@ write_thread_id(int fd, PyThreadState *tstate, int is_current) PUTS(fd, " (most recent call first):\n"); } +/* Write an error string and also return it at the same time. */ +static const char* +dump_error(int fd, const char *msg) +{ + PUTS(fd, msg); + return msg; +} + /* Dump the traceback of all Python threads into fd. Use write() to write the traceback and retry if write() is interrupted by a signal (failed with EINTR), but don't call the Python signal handler. @@ -1264,11 +1273,11 @@ write_thread_id(int fd, PyThreadState *tstate, int is_current) The caller is responsible to call PyErr_CheckSignals() to call Python signal handlers if signals were received. */ const char* _Py_NO_SANITIZE_THREAD -_Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, - PyThreadState *current_tstate) +PyUnstable_DumpTracebackThreads(int fd, PyInterpreterState *interp, + PyThreadState *current_tstate) { if (current_tstate == NULL) { - /* _Py_DumpTracebackThreads() is called from signal handlers by + /* PyUnstable_DumpTracebackThreads() is called from signal handlers by faulthandler. SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL are synchronous signals @@ -1283,7 +1292,7 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, } if (current_tstate != NULL && tstate_is_freed(current_tstate)) { - return "tstate is freed"; + return dump_error(fd, "tstate is freed"); } if (interp == NULL) { @@ -1291,7 +1300,7 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, interp = _PyGILState_GetInterpreterStateUnsafe(); if (interp == NULL) { /* We need the interpreter state to get Python threads */ - return "unable to get the interpreter state"; + return dump_error(fd, "unable to get the interpreter state"); } } else { @@ -1301,13 +1310,13 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, assert(interp != NULL); if (interp_is_freed(interp)) { - return "interp is freed"; + return dump_error(fd, "interp is freed"); } /* Get the current interpreter from the current thread */ PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); if (tstate == NULL) - return "unable to get the thread head state"; + return dump_error(fd, "unable to get the thread head state"); /* Dump the traceback of each thread */ unsigned int nthreads = 0; diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index 4d92b083026b27..53d7b8fe32f89e 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -48,6 +48,7 @@ SUFFIXES_DOCUMENTATION = frozenset({".rst", ".md"}) ANDROID_DIRS = frozenset({"Android"}) +EMSCRIPTEN_DIRS = frozenset({Path("Platforms", "emscripten")}) IOS_DIRS = frozenset({"Apple", "iOS"}) MACOS_DIRS = frozenset({"Mac"}) WASI_DIRS = frozenset({Path("Platforms", "WASI")}) @@ -98,6 +99,9 @@ Path("Modules/pyexpat.c"), # zipfile Path("Lib/zipfile/"), + # zoneinfo + Path("Lib/zoneinfo/"), + Path("Modules/_zoneinfo.c"), }) @@ -107,6 +111,7 @@ class Outputs: run_ci_fuzz: bool = False run_ci_fuzz_stdlib: bool = False run_docs: bool = False + run_emscripten: bool = False run_ios: bool = False run_macos: bool = False run_tests: bool = False @@ -126,6 +131,7 @@ def compute_changes() -> None: # Otherwise, just run the tests outputs = Outputs( run_android=True, + run_emscripten=True, run_ios=True, run_macos=True, run_tests=True, @@ -196,6 +202,8 @@ def get_file_platform(file: Path) -> str | None: return "ios" if first_part in ANDROID_DIRS: return "android" + if len(file.parts) >= 2 and Path(*file.parts[:2]) in EMSCRIPTEN_DIRS: + return "emscripten" if len(file.parts) >= 2 and Path(*file.parts[:2]) in WASI_DIRS: return "wasi" return None @@ -230,7 +238,7 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: run_tests = run_ci_fuzz = run_ci_fuzz_stdlib = run_windows_tests = True has_platform_specific_change = False continue - if file.name == "reusable-docs.yml": + if file.name in ("reusable-docs.yml", "reusable-check-html-ids.yml"): run_docs = True continue if file.name == "reusable-windows.yml": @@ -244,6 +252,10 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: run_tests = True platforms_changed.add("macos") continue + if file.name == "reusable-emscripten.yml": + run_tests = True + platforms_changed.add("emscripten") + continue if file.name == "reusable-wasi.yml": run_tests = True platforms_changed.add("wasi") @@ -284,18 +296,21 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: if run_tests: if not has_platform_specific_change or not platforms_changed: run_android = True + run_emscripten = True run_ios = True run_macos = True run_ubuntu = True run_wasi = True else: run_android = "android" in platforms_changed + run_emscripten = "emscripten" in platforms_changed run_ios = "ios" in platforms_changed run_macos = "macos" in platforms_changed run_ubuntu = False run_wasi = "wasi" in platforms_changed else: run_android = False + run_emscripten = False run_ios = False run_macos = False run_ubuntu = False @@ -306,6 +321,7 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: run_ci_fuzz=run_ci_fuzz, run_ci_fuzz_stdlib=run_ci_fuzz_stdlib, run_docs=run_docs, + run_emscripten=run_emscripten, run_ios=run_ios, run_macos=run_macos, run_tests=run_tests, diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index cbec0bf262f0e0..d2489387f46caa 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -351,7 +351,7 @@ Objects/obmalloc.c - obmalloc_state_initialized - Objects/typeobject.c - name_op - Objects/typeobject.c - slotdefs - # It initialized only once when main interpeter starts -Objects/typeobject.c - slotdefs_name_counts - +Objects/typeobject.c - slotdefs_dups - Objects/unicodeobject.c - stripfuncnames - Objects/unicodeobject.c - utf7_category - Objects/unicodeobject.c unicode_decode_call_errorhandler_wchar argparse - diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 89f843644329ec..6ba9c43ef1f0c3 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -714,7 +714,9 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "trigger_backoff_counter", "_PyThreadState_PopCStackRefSteal", "doesnt_escape", - + "_Py_GatherStats_GetIter", + "_PyStolenTuple_Free", + "PyObject_GC_UnTrack", ) diff --git a/Tools/check-c-api-docs/ignored_c_api.txt b/Tools/check-c-api-docs/ignored_c_api.txt index e464162c52a371..f3a3612b84947a 100644 --- a/Tools/check-c-api-docs/ignored_c_api.txt +++ b/Tools/check-c-api-docs/ignored_c_api.txt @@ -28,19 +28,7 @@ PyExpat_CAPSULE_NAME # pyport.h PYLONG_BITS_IN_DIGIT PY_DWORD_MAX -PY_FORMAT_SIZE_T -PY_INT32_T -PY_INT64_T -PY_LLONG_MAX -PY_LLONG_MIN -PY_LONG_LONG -PY_SIZE_MAX -PY_UINT32_T -PY_UINT64_T -PY_ULLONG_MAX PY_BIG_ENDIAN -# unicodeobject.h -Py_UNICODE_SIZE # cpython/methodobject.h PyCFunction_GET_CLASS # cpython/compile.h diff --git a/Tools/ftscalingbench/ftscalingbench.py b/Tools/ftscalingbench/ftscalingbench.py index bcbd61f601a7d3..a3d87e1f855dcb 100644 --- a/Tools/ftscalingbench/ftscalingbench.py +++ b/Tools/ftscalingbench/ftscalingbench.py @@ -285,6 +285,15 @@ def deepcopy(): for i in range(40 * WORK_SCALE): copy.deepcopy(x) +@register_benchmark +def setattr_non_interned(): + prefix = "prefix" + obj = MyObject() + for _ in range(1000 * WORK_SCALE): + setattr(obj, f"{prefix}_a", None) + setattr(obj, f"{prefix}_b", None) + setattr(obj, f"{prefix}_c", None) + def bench_one_thread(func): t0 = time.perf_counter_ns() diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 8b68c1e8636af7..a4aaacdf41249d 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -42,9 +42,19 @@ async def _run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str async with _CORES: if echo: print(shlex.join(command)) + + if os.name == "nt": + # When building with /p:PlatformToolset=ClangCL, the VS build + # system puts that clang's include path into INCLUDE. The JIT's + # clang may be a different version, and mismatched headers cause + # build errors. See https://github.com/python/cpython/issues/146210. + env = os.environ.copy() + env.pop("INCLUDE", None) + else: + env = None try: process = await asyncio.create_subprocess_exec( - *command, stdout=subprocess.PIPE + *command, stdout=subprocess.PIPE, env=env ) except FileNotFoundError: return None diff --git a/Tools/jit/_optimizers.py b/Tools/jit/_optimizers.py index 83c878d8fe205b..ef28e0c0ddeac8 100644 --- a/Tools/jit/_optimizers.py +++ b/Tools/jit/_optimizers.py @@ -162,6 +162,7 @@ class Optimizer: label_prefix: str symbol_prefix: str re_global: re.Pattern[str] + frame_pointers: bool # The first block in the linked list: _root: _Block = dataclasses.field(init=False, default_factory=_Block) _labels: dict[str, _Block] = dataclasses.field(init=False, default_factory=dict) @@ -193,6 +194,7 @@ class Optimizer: _re_small_const_1 = _RE_NEVER_MATCH _re_small_const_2 = _RE_NEVER_MATCH const_reloc = "" + _frame_pointer_modify: typing.ClassVar[re.Pattern[str]] = _RE_NEVER_MATCH def __post_init__(self) -> None: # Split the code into a linked list of basic blocks. A basic block is an @@ -553,6 +555,16 @@ def _small_const_2(self, inst: Instruction) -> tuple[str, Instruction | None]: def _small_consts_match(self, inst1: Instruction, inst2: Instruction) -> bool: raise NotImplementedError() + def _validate(self) -> None: + for block in self._blocks(): + if not block.instructions: + continue + for inst in block.instructions: + if self.frame_pointers: + assert ( + self._frame_pointer_modify.match(inst.text) is None + ), "Frame pointer should not be modified" + def run(self) -> None: """Run this optimizer.""" self._insert_continue_label() @@ -565,6 +577,7 @@ def run(self) -> None: self._remove_unreachable() self._fixup_external_labels() self._fixup_constants() + self._validate() self.path.write_text(self._body()) @@ -595,6 +608,7 @@ class OptimizerAArch64(Optimizer): # pylint: disable = too-few-public-methods r"\s*(?Pldr)\s+.*(?P_JIT_OP(ARG|ERAND(0|1))_(16|32)).*" ) const_reloc = "CUSTOM_AARCH64_CONST" + _frame_pointer_modify = re.compile(r"\s*stp\s+x29.*") def _get_reg(self, inst: Instruction) -> str: _, rest = inst.text.split(inst.name) @@ -649,4 +663,5 @@ class OptimizerX86(Optimizer): # pylint: disable = too-few-public-methods # https://www.felixcloutier.com/x86/jmp _re_jump = re.compile(r"\s*jmp\s+(?P[\w.]+)") # https://www.felixcloutier.com/x86/ret - _re_return = re.compile(r"\s*ret\b") + _re_return = re.compile(r"\s*retq?\b") + _frame_pointer_modify = re.compile(r"\s*movq?\s+%(\w+),\s+%rbp.*") diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 39be353ec30858..787fcf53260f3d 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -51,6 +51,7 @@ class _Target(typing.Generic[_S, _R]): debug: bool = False verbose: bool = False cflags: str = "" + frame_pointers: bool = False llvm_version: str = _llvm._LLVM_VERSION known_symbols: dict[str, int] = dataclasses.field(default_factory=dict) pyconfig_dir: pathlib.Path = pathlib.Path.cwd().resolve() @@ -174,19 +175,25 @@ async def _compile( "-o", f"{s}", f"{c}", - *self.args, - # Allow user-provided CFLAGS to override any defaults - *shlex.split(self.cflags), ] + is_shim = opname == "shim" + if self.frame_pointers: + frame_pointer = "all" if is_shim else "reserved" + args_s += ["-Xclang", f"-mframe-pointer={frame_pointer}"] + args_s += self.args + # Allow user-provided CFLAGS to override any defaults + args_s += shlex.split(self.cflags) await _llvm.run( "clang", args_s, echo=self.verbose, llvm_version=self.llvm_version ) - self.optimizer( - s, - label_prefix=self.label_prefix, - symbol_prefix=self.symbol_prefix, - re_global=self.re_global, - ).run() + if not is_shim: + self.optimizer( + s, + label_prefix=self.label_prefix, + symbol_prefix=self.symbol_prefix, + re_global=self.re_global, + frame_pointers=self.frame_pointers, + ).run() args_o = [f"--target={self.triple}", "-c", "-o", f"{o}", f"{s}"] await _llvm.run( "clang", args_o, echo=self.verbose, llvm_version=self.llvm_version @@ -589,12 +596,16 @@ def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: # -mno-outline-atomics: Keep intrinsics from being emitted. args = ["-fpic", "-mno-outline-atomics", "-fno-plt"] optimizer = _optimizers.OptimizerAArch64 - target = _ELF(host, condition, args=args, optimizer=optimizer) + target = _ELF( + host, condition, args=args, optimizer=optimizer, frame_pointers=True + ) elif re.fullmatch(r"i686-pc-windows-msvc", host): host = "i686-pc-windows-msvc" condition = "defined(_M_IX86)" # -Wno-ignored-attributes: __attribute__((preserve_none)) is not supported here. - args = ["-DPy_NO_ENABLE_SHARED", "-Wno-ignored-attributes"] + # -mno-sse: Use x87 FPU instead of SSE for float math. The COFF32 + # stencil converter cannot handle _xmm register references. + args = ["-DPy_NO_ENABLE_SHARED", "-Wno-ignored-attributes", "-mno-sse"] optimizer = _optimizers.OptimizerX86 target = _COFF32(host, condition, args=args, optimizer=optimizer) elif re.fullmatch(r"x86_64-apple-darwin.*", host): @@ -613,7 +624,9 @@ def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: condition = "defined(__x86_64__) && defined(__linux__)" args = ["-fno-pic", "-mcmodel=medium", "-mlarge-data-threshold=0", "-fno-plt"] optimizer = _optimizers.OptimizerX86 - target = _ELF(host, condition, args=args, optimizer=optimizer) + target = _ELF( + host, condition, args=args, optimizer=optimizer, frame_pointers=True + ) else: raise ValueError(host) return target diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index 3b4507c6771b69..48207e5330fa90 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -429,9 +429,11 @@ class BuildOpenSSL(AbstractBuilder): def _post_install(self): if self.version.startswith("3."): self._post_install_3xx() + elif self.version.startswith("4."): + self._post_install_4xx() def _build_src(self, config_args=()): - if self.version.startswith("3."): + if self.version.startswith(("3.", "4.")): config_args += ("enable-fips",) super()._build_src(config_args) @@ -447,6 +449,9 @@ def _post_install_3xx(self): lib64 = self.lib_dir + "64" os.symlink(lib64, self.lib_dir) + def _post_install_4xx(self): + self._post_install_3xx() + @property def short_version(self): """Short version for OpenSSL download URL""" diff --git a/configure b/configure index 23f24d51c79e1a..1d070504d75021 100755 --- a/configure +++ b/configure @@ -1133,6 +1133,7 @@ with_pymalloc_hugepages with_c_locale_coercion with_valgrind with_dtrace +enable_epoll with_libm with_libc enable_big_digits @@ -1856,6 +1857,7 @@ Optional Features: see Doc/library/sqlite3.rst (default is no) --enable-ipv6 enable ipv6 (with ipv4) support, see Doc/library/socket.rst (default is yes if supported) + --disable-epoll disable epoll (default is yes if supported) --enable-big-digits[=15|30] use big digits (30 or 15 bits) for Python longs (default is 30)] @@ -4149,6 +4151,9 @@ then *-*-darwin*) ac_sys_system=Darwin ;; + *-gnu) + ac_sys_system=GNU + ;; *-*-vxworks*) ac_sys_system=VxWorks ;; @@ -4389,7 +4394,7 @@ then : yes) case $ac_sys_system in Darwin) enableval=/Library/Frameworks ;; - iOS) enableval=Apple/iOS/Frameworks/\$\(MULTIARCH\) ;; + iOS) enableval=Platforms/Apple/iOS/Frameworks/\$\(MULTIARCH\) ;; *) as_fn_error $? "Unknown platform for framework build" "$LINENO" 5 esac esac @@ -4500,9 +4505,9 @@ then : prefix=$PYTHONFRAMEWORKPREFIX PYTHONFRAMEWORKINSTALLNAMEPREFIX="@rpath/$PYTHONFRAMEWORKDIR" - RESSRCDIR=Apple/iOS/Resources + RESSRCDIR=Platforms/Apple/iOS/Resources - ac_config_files="$ac_config_files Apple/iOS/Resources/Info.plist" + ac_config_files="$ac_config_files Platforms/Apple/iOS/Resources/Info.plist" ;; *) @@ -4619,6 +4624,9 @@ if test "$cross_compiling" = yes; then _host_ident=$host_cpu esac ;; + *-gnu) + _host_ident=$host_cpu + ;; *-*-cygwin*) _host_ident= ;; @@ -9690,8 +9698,8 @@ fi as_fn_append LDFLAGS_NODIST " -sWASM_BIGINT" as_fn_append LINKFORSHARED " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" - as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY" - as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,_PyGILState_GetThisThreadState,__Py_DumpTraceback,__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET" + as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY,ERRNO_CODES" + as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,_PyGILState_GetThisThreadState,__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET" as_fn_append LINKFORSHARED " -sSTACK_SIZE=5MB" as_fn_append LINKFORSHARED " -sTEXTDECODER=2" @@ -21000,6 +21008,29 @@ fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --disable-epoll" >&5 +printf %s "checking for --disable-epoll... " >&6; } +# Check whether --enable-epoll was given. +if test ${enable_epoll+y} +then : + enableval=$enable_epoll; if test "x$enable_epoll" = xno +then : + disable_epoll=yes +else case e in #( + e) disable_epoll=no ;; +esac +fi +else case e in #( + e) disable_epoll=no + ;; +esac +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $disable_epoll" >&5 +printf "%s\n" "$disable_epoll" >&6; } +if test "$disable_epoll" = "no" +then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for epoll_create" >&5 printf %s "checking for epoll_create... " >&6; } @@ -21081,6 +21112,8 @@ fi +fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for kqueue" >&5 @@ -35618,7 +35651,7 @@ do "Mac/PythonLauncher/Makefile") CONFIG_FILES="$CONFIG_FILES Mac/PythonLauncher/Makefile" ;; "Mac/Resources/framework/Info.plist") CONFIG_FILES="$CONFIG_FILES Mac/Resources/framework/Info.plist" ;; "Mac/Resources/app/Info.plist") CONFIG_FILES="$CONFIG_FILES Mac/Resources/app/Info.plist" ;; - "Apple/iOS/Resources/Info.plist") CONFIG_FILES="$CONFIG_FILES Apple/iOS/Resources/Info.plist" ;; + "Platforms/Apple/iOS/Resources/Info.plist") CONFIG_FILES="$CONFIG_FILES Platforms/Apple/iOS/Resources/Info.plist" ;; "Makefile.pre") CONFIG_FILES="$CONFIG_FILES Makefile.pre" ;; "Misc/python.pc") CONFIG_FILES="$CONFIG_FILES Misc/python.pc" ;; "Misc/python-embed.pc") CONFIG_FILES="$CONFIG_FILES Misc/python-embed.pc" ;; @@ -36278,4 +36311,3 @@ if test "$ac_cv_header_stdatomic_h" != "yes"; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: Your compiler or platform does have a working C11 stdatomic.h. A future version of Python may require stdatomic.h." >&5 printf "%s\n" "$as_me: Your compiler or platform does have a working C11 stdatomic.h. A future version of Python may require stdatomic.h." >&6;} fi - diff --git a/configure.ac b/configure.ac index 635fce3f2e6fad..44754fc0353600 100644 --- a/configure.ac +++ b/configure.ac @@ -342,6 +342,9 @@ then *-*-darwin*) ac_sys_system=Darwin ;; + *-gnu) + ac_sys_system=GNU + ;; *-*-vxworks*) ac_sys_system=VxWorks ;; @@ -568,7 +571,7 @@ AC_ARG_ENABLE([framework], yes) case $ac_sys_system in Darwin) enableval=/Library/Frameworks ;; - iOS) enableval=Apple/iOS/Frameworks/\$\(MULTIARCH\) ;; + iOS) enableval=Platforms/Apple/iOS/Frameworks/\$\(MULTIARCH\) ;; *) AC_MSG_ERROR([Unknown platform for framework build]) esac esac @@ -675,9 +678,9 @@ AC_ARG_ENABLE([framework], prefix=$PYTHONFRAMEWORKPREFIX PYTHONFRAMEWORKINSTALLNAMEPREFIX="@rpath/$PYTHONFRAMEWORKDIR" - RESSRCDIR=Apple/iOS/Resources + RESSRCDIR=Platforms/Apple/iOS/Resources - AC_CONFIG_FILES([Apple/iOS/Resources/Info.plist]) + AC_CONFIG_FILES([Platforms/Apple/iOS/Resources/Info.plist]) ;; *) AC_MSG_ERROR([Unknown platform for framework build]) @@ -780,6 +783,9 @@ if test "$cross_compiling" = yes; then _host_ident=$host_cpu esac ;; + *-gnu) + _host_ident=$host_cpu + ;; *-*-cygwin*) _host_ident= ;; @@ -2359,8 +2365,8 @@ AS_CASE([$ac_sys_system], dnl Include file system support AS_VAR_APPEND([LINKFORSHARED], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) - AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY"]) - AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,_PyGILState_GetThisThreadState,__Py_DumpTraceback,__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY,ERRNO_CODES"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,_PyGILState_GetThisThreadState,__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET"]) AS_VAR_APPEND([LINKFORSHARED], [" -sSTACK_SIZE=5MB"]) dnl Avoid bugs in JS fallback string decoding path AS_VAR_APPEND([LINKFORSHARED], [" -sTEXTDECODER=2"]) @@ -5383,8 +5389,20 @@ PY_CHECK_FUNC([symlink], [@%:@include ]) PY_CHECK_FUNC([fchdir], [@%:@include ]) PY_CHECK_FUNC([fsync], [@%:@include ]) PY_CHECK_FUNC([fdatasync], [@%:@include ]) -PY_CHECK_FUNC([epoll_create], [@%:@include ], [HAVE_EPOLL]) -PY_CHECK_FUNC([epoll_create1], [@%:@include ]) + +AC_MSG_CHECKING([for --disable-epoll]) +AC_ARG_ENABLE([epoll], + [AS_HELP_STRING([--disable-epoll], [disable epoll (default is yes if supported)])], + [AS_VAR_IF([enable_epoll], [no], [disable_epoll=yes], [disable_epoll=no])], + [disable_epoll=no] +) +AC_MSG_RESULT([$disable_epoll]) +if test "$disable_epoll" = "no" +then + PY_CHECK_FUNC([epoll_create], [@%:@include ], [HAVE_EPOLL]) + PY_CHECK_FUNC([epoll_create1], [@%:@include ]) +fi + PY_CHECK_FUNC([kqueue],[ #include #include diff --git a/iOS/Resources/bin/arm64-apple-ios-ar b/iOS/Resources/bin/arm64-apple-ios-ar deleted file mode 100755 index 3cf3eb218741fa..00000000000000 --- a/iOS/Resources/bin/arm64-apple-ios-ar +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -xcrun --sdk iphoneos${IOS_SDK_VERSION} ar "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-clang b/iOS/Resources/bin/arm64-apple-ios-clang deleted file mode 100755 index f50d5b5142fc76..00000000000000 --- a/iOS/Resources/bin/arm64-apple-ios-clang +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-clang++ b/iOS/Resources/bin/arm64-apple-ios-clang++ deleted file mode 100755 index 0794731d7dcbda..00000000000000 --- a/iOS/Resources/bin/arm64-apple-ios-clang++ +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -xcrun --sdk iphoneos${IOS_SDK_VERSION} clang++ -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-cpp b/iOS/Resources/bin/arm64-apple-ios-cpp deleted file mode 100755 index 24fa1506bab827..00000000000000 --- a/iOS/Resources/bin/arm64-apple-ios-cpp +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} -E "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-ar b/iOS/Resources/bin/arm64-apple-ios-simulator-ar deleted file mode 100755 index b836b6db9025bb..00000000000000 --- a/iOS/Resources/bin/arm64-apple-ios-simulator-ar +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-clang b/iOS/Resources/bin/arm64-apple-ios-simulator-clang deleted file mode 100755 index 4891a00876e0bd..00000000000000 --- a/iOS/Resources/bin/arm64-apple-ios-simulator-clang +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ deleted file mode 100755 index 58b2a5f6f18c2b..00000000000000 --- a/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-cpp b/iOS/Resources/bin/arm64-apple-ios-simulator-cpp deleted file mode 100755 index c9df94e8b7c837..00000000000000 --- a/iOS/Resources/bin/arm64-apple-ios-simulator-cpp +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator -E "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-strip b/iOS/Resources/bin/arm64-apple-ios-simulator-strip deleted file mode 100755 index fd59d309b73a20..00000000000000 --- a/iOS/Resources/bin/arm64-apple-ios-simulator-strip +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} strip -arch arm64 "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-strip b/iOS/Resources/bin/arm64-apple-ios-strip deleted file mode 100755 index 75e823a3d02d61..00000000000000 --- a/iOS/Resources/bin/arm64-apple-ios-strip +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -xcrun --sdk iphoneos${IOS_SDK_VERSION} strip -arch arm64 "$@" diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-ar b/iOS/Resources/bin/x86_64-apple-ios-simulator-ar deleted file mode 100755 index b836b6db9025bb..00000000000000 --- a/iOS/Resources/bin/x86_64-apple-ios-simulator-ar +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar "$@" diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang deleted file mode 100755 index f4739a7b945d01..00000000000000 --- a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@" diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ deleted file mode 100755 index c348ae4c10395b..00000000000000 --- a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@" diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp b/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp deleted file mode 100755 index 6d7f8084c9fdcc..00000000000000 --- a/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator -E "$@" diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-strip b/iOS/Resources/bin/x86_64-apple-ios-simulator-strip deleted file mode 100755 index c5cfb28929195a..00000000000000 --- a/iOS/Resources/bin/x86_64-apple-ios-simulator-strip +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} strip -arch x86_64 "$@"