From 91e8a144f2fb406146ff22767b1d2d8af9c22c74 Mon Sep 17 00:00:00 2001 From: Shubham Dhal Date: Mon, 25 May 2026 11:03:43 +0530 Subject: [PATCH 1/2] ci(min-deps): promote min-deps to first-class CI alongside latest-deps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - setup-python-deps + warmDepsCache: cache key hash includes requirements.lowest-direct.txt; warm job creates min-deps + verify-min-deps envs and downloads the lowest-direct closure into the wheelhouse so fork PRs can run min-deps tests fully offline from cache. - main-min-deps.yml (renamed from min-deps-test-fast.yml): mirrors main.yml shape — cache-restore + JFrog fallback, concurrency with cancel-in-progress, paths-ignore for *.md. Fork-skip guards removed; cache makes forks work. - integration-min-deps.yml (renamed from min-deps-test-slow.yml): adds workflow_dispatch with pr_numbers / git_ref inputs, full target parser, and a report-status job that posts a result comment back to the PR — mirroring integration.yml for the slash-command path. - integration-trigger.yml: parses the first arg after /integration-test. /integration-test min-deps routes to integration-min-deps.yml; anything else falls through to integration.yml (free-form trailing context). --- .github/actions/setup-python-deps/action.yml | 4 +- ...test-slow.yml => integration-min-deps.yml} | 154 +++++++++++++++--- .github/workflows/integration-trigger.yml | 27 ++- ...n-deps-test-fast.yml => main-min-deps.yml} | 42 +++-- .github/workflows/warmDepsCache.yml | 9 +- 5 files changed, 189 insertions(+), 47 deletions(-) rename .github/workflows/{min-deps-test-slow.yml => integration-min-deps.yml} (75%) rename .github/workflows/{min-deps-test-fast.yml => main-min-deps.yml} (69%) diff --git a/.github/actions/setup-python-deps/action.yml b/.github/actions/setup-python-deps/action.yml index 83b9593b4..fd2684604 100644 --- a/.github/actions/setup-python-deps/action.yml +++ b/.github/actions/setup-python-deps/action.yml @@ -19,8 +19,8 @@ runs: ~/.cache/uv ~/.cache/pip ~/.cache/pip-wheelhouse - key: python-deps-${{ hashFiles('uv.lock', 'pyproject.toml') }}-latest - restore-keys: python-deps-${{ hashFiles('uv.lock', 'pyproject.toml') }}- + key: python-deps-${{ hashFiles('uv.lock', 'requirements.lowest-direct.txt', 'pyproject.toml') }}-latest + restore-keys: python-deps-${{ hashFiles('uv.lock', 'requirements.lowest-direct.txt', 'pyproject.toml') }}- - name: Restore pre-commit cache uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 diff --git a/.github/workflows/min-deps-test-slow.yml b/.github/workflows/integration-min-deps.yml similarity index 75% rename from .github/workflows/min-deps-test-slow.yml rename to .github/workflows/integration-min-deps.yml index a4efebbe3..be0ce8ea3 100644 --- a/.github/workflows/min-deps-test-slow.yml +++ b/.github/workflows/integration-min-deps.yml @@ -1,4 +1,4 @@ -# Min-Deps Integration Tests (slow) +# Integration Tests (Min-Deps) for dbt-databricks # # Sharded functional tests against the lowest-direct dependency resolution # committed in requirements.lowest-direct.txt. Mirrors integration.yml's @@ -6,10 +6,32 @@ # timing, sharded execution per profile, gather-shards verification) but # routes every pytest invocation through the `min-deps:` hatch env. # -# Triggering: schedule only. - -name: Min-Deps Tests (slow) +# Triggering: +# 1. On a PR (internal OR fork): a maintainer comments `/integration-test min-deps`. +# The integration-trigger workflow validates the comment author and +# dispatches this workflow with the PR number. The run posts a result +# comment back to the PR when the matrix completes. +# 2. Manually from the Actions tab (workflow_dispatch): +# - One PR (e.g. "100") OR comma-separated list ("100,200,300") in pr_numbers. +# - Or a git_ref for ad-hoc testing. +# 3. Nightly on `main` at 19:30 UTC (2h before integration.yml's 21:30 nightly). +# The `prepare` job short-circuits if the current main SHA has already had a +# successful min-deps integration run, emitting an empty targets array so +# the matrix jobs skip cleanly. + +name: Integration Tests (Min-Deps) on: + workflow_dispatch: + inputs: + pr_numbers: + description: "PR number(s) to test — single PR or comma-separated for batch (e.g. '100' or '100,200,300')" + required: false + type: string + git_ref: + description: "Git ref (branch/tag/commit) to test — used only when pr_numbers is empty" + required: false + type: string + schedule: - cron: "30 19 * * *" # 19:30 UTC, 2h before integration.yml's nightly @@ -18,7 +40,7 @@ permissions: contents: read concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.event.inputs.pr_numbers || github.event.inputs.git_ref || github.ref }} cancel-in-progress: true defaults: @@ -43,23 +65,46 @@ jobs: id: parse shell: bash env: + EVENT_NAME: ${{ github.event_name }} + INPUT_PR_NUMBERS: ${{ github.event.inputs.pr_numbers }} + INPUT_GIT_REF: ${{ github.event.inputs.git_ref }} DEFAULT_REF: ${{ github.ref }} GH_TOKEN: ${{ github.token }} run: | set -euo pipefail entry() { printf '{"pr":"%s","ref":"%s"}' "$1" "$2"; } targets="[" - # Nightly skip-if-unchanged: if this main SHA already has a green - # min-deps-slow run, emit empty targets so the matrix jobs skip. - already_tested=$(curl -sfS \ - -H "Authorization: Bearer $GH_TOKEN" \ - -H "Accept: application/vnd.github+json" \ - "https://api.github.com/repos/$GITHUB_REPOSITORY/actions/workflows/min-deps-test-slow.yml/runs?branch=main&status=success&head_sha=$GITHUB_SHA" \ - | jq -r '.total_count // 0') - if [[ "$already_tested" -gt 0 ]]; then - echo "Nightly skip: main @ $GITHUB_SHA already has $already_tested successful run(s)." + if [[ "$EVENT_NAME" == "schedule" ]]; then + # Nightly skip-if-unchanged: if this main SHA already has a green + # min-deps integration run, emit empty targets so the matrix jobs skip. + already_tested=$(curl -sfS \ + -H "Authorization: Bearer $GH_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/$GITHUB_REPOSITORY/actions/workflows/integration-min-deps.yml/runs?branch=main&status=success&head_sha=$GITHUB_SHA" \ + | jq -r '.total_count // 0') + if [[ "$already_tested" -gt 0 ]]; then + echo "Nightly skip: main @ $GITHUB_SHA already has $already_tested successful run(s)." + else + targets+=$(entry "nightly" "$DEFAULT_REF") + fi + elif [[ -n "${INPUT_PR_NUMBERS//[[:space:]]/}" ]]; then + first=1 + IFS=',' read -ra prs <<< "$INPUT_PR_NUMBERS" + for pr in "${prs[@]}"; do + pr_trimmed="${pr//[[:space:]]/}" + [[ -z "$pr_trimmed" ]] && continue + if [[ ! "$pr_trimmed" =~ ^[0-9]+$ ]]; then + echo "::error::Invalid PR number '$pr_trimmed' in pr_numbers='$INPUT_PR_NUMBERS' — expected digits, comma-separated." + exit 1 + fi + [[ $first -eq 0 ]] && targets+="," + first=0 + targets+=$(entry "$pr_trimmed" "refs/pull/$pr_trimmed/head") + done + elif [[ -n "${INPUT_GIT_REF//[[:space:]]/}" ]]; then + targets+=$(entry "manual" "$INPUT_GIT_REF") else - targets+=$(entry "nightly" "$DEFAULT_REF") + targets+=$(entry "manual" "$DEFAULT_REF") fi targets+="]" echo "targets=$targets" >> "$GITHUB_OUTPUT" @@ -97,7 +142,12 @@ jobs: with: ref: ${{ matrix.target.ref }} - - name: Setup JFrog PyPI Proxy + - name: Setup Python Dependencies + id: deps + uses: ./.github/actions/setup-python-deps + + - name: Setup JFrog PyPI Proxy (fallback) + if: steps.deps.outputs.cache-hit != 'true' uses: ./.github/actions/setup-jfrog-pypi - name: Set up python @@ -193,7 +243,12 @@ jobs: with: ref: ${{ matrix.target.ref }} - - name: Setup JFrog PyPI Proxy + - name: Setup Python Dependencies + id: deps + uses: ./.github/actions/setup-python-deps + + - name: Setup JFrog PyPI Proxy (fallback) + if: steps.deps.outputs.cache-hit != 'true' uses: ./.github/actions/setup-jfrog-pypi - name: Set up python @@ -287,7 +342,12 @@ jobs: with: ref: ${{ matrix.target.ref }} - - name: Setup JFrog PyPI Proxy + - name: Setup Python Dependencies + id: deps + uses: ./.github/actions/setup-python-deps + + - name: Setup JFrog PyPI Proxy (fallback) + if: steps.deps.outputs.cache-hit != 'true' uses: ./.github/actions/setup-jfrog-pypi - name: Set up python @@ -379,7 +439,12 @@ jobs: with: ref: ${{ matrix.target.ref }} - - name: Setup JFrog PyPI Proxy + - name: Setup Python Dependencies + id: deps + uses: ./.github/actions/setup-python-deps + + - name: Setup JFrog PyPI Proxy (fallback) + if: steps.deps.outputs.cache-hit != 'true' uses: ./.github/actions/setup-jfrog-pypi - name: Set up python @@ -511,3 +576,54 @@ jobs: echo "::error::One or more shard-verification invariants failed." exit 1 fi + + # Posts a per-job pass/fail summary comment back to the PR when dispatched + # with a single PR number (the slash-command path). Skipped for batch + # dispatches and for schedule / git_ref runs. + report-status: + needs: + - run-uc-cluster-e2e-tests + - run-sqlwarehouse-e2e-tests + - run-cluster-e2e-tests + - gather-shards + if: | + always() && + github.event_name == 'workflow_dispatch' && + inputs.pr_numbers != '' && + !contains(inputs.pr_numbers, ',') + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Post result comment + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + env: + PR_NUMBER: ${{ inputs.pr_numbers }} + UC_RESULT: ${{ needs.run-uc-cluster-e2e-tests.result }} + SQLW_RESULT: ${{ needs.run-sqlwarehouse-e2e-tests.result }} + CLUSTER_RESULT: ${{ needs.run-cluster-e2e-tests.result }} + GATHER_RESULT: ${{ needs.gather-shards.result }} + with: + script: | + const ICONS = { success: ':white_check_mark:', skipped: ':fast_forward:' }; + const results = { + 'UC cluster': process.env.UC_RESULT, + 'SQL warehouse': process.env.SQLW_RESULT, + 'All-purpose cluster': process.env.CLUSTER_RESULT, + 'Shard coverage': process.env.GATHER_RESULT, + }; + const line = Object.entries(results) + .map(([name, r]) => `${name} ${ICONS[r] || ':x:'} ${r}`) + .join(' · '); + const runUrl = + `https://github.com/${context.repo.owner}/${context.repo.repo}` + + `/actions/runs/${context.runId}`; + const prNumber = process.env.PR_NUMBER.trim(); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: parseInt(prNumber, 10), + body: `Min-deps integration results for PR #${prNumber} — ${line}\n\n[Run details](${runUrl}).`, + }); diff --git a/.github/workflows/integration-trigger.yml b/.github/workflows/integration-trigger.yml index bae813717..a509867eb 100644 --- a/.github/workflows/integration-trigger.yml +++ b/.github/workflows/integration-trigger.yml @@ -1,9 +1,15 @@ # Integration Test Trigger # -# Listens for `/integration-test` PR comments and dispatches the integration -# test workflow against the PR head. Works uniformly for internal and forked -# PRs because the dispatch runs in the main repo context (which has access to -# the Databricks secrets). +# Listens for `/integration-test` PR comments and dispatches the appropriate +# integration test workflow against the PR head. Works uniformly for internal +# and forked PRs because the dispatch runs in the main repo context (which +# has access to the Databricks secrets). +# +# Routing on the first whitespace-separated arg after `/integration-test`: +# /integration-test → integration.yml (latest deps) +# /integration-test min-deps → integration-min-deps.yml (lowest-direct lock) +# /integration-test → integration.yml (trailing text +# is free-form context, e.g. retry notes) # # Authorization: only users whose author_association is OWNER, MEMBER, or # COLLABORATOR can trigger a run. Anyone else gets a reply comment explaining @@ -22,8 +28,7 @@ permissions: jobs: dispatch: # Exact command match, or the command followed by whitespace (so maintainers - # can add context like `/integration-test please retry the sqlw flake`) — - # never matches `/integration-test-foo`. + # can add context or modifiers) — never matches `/integration-test-foo`. if: | github.event.issue.pull_request && (github.event.comment.body == '/integration-test' || startsWith(github.event.comment.body, '/integration-test ')) && @@ -47,22 +52,26 @@ jobs: uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 with: script: | + const rest = context.payload.comment.body.slice('/integration-test'.length).trim(); + const isMinDeps = rest === 'min-deps' || rest.startsWith('min-deps '); + const workflowId = isMinDeps ? 'integration-min-deps.yml' : 'integration.yml'; + const label = isMinDeps ? 'Min-deps integration tests' : 'Integration tests'; const prNumber = String(context.payload.issue.number); await github.rest.actions.createWorkflowDispatch({ owner: context.repo.owner, repo: context.repo.repo, - workflow_id: 'integration.yml', + workflow_id: workflowId, ref: 'main', inputs: { pr_numbers: prNumber }, }); const actionsUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}` + - `/actions/workflows/integration.yml`; + `/actions/workflows/${workflowId}`; await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.payload.issue.number, - body: `Integration tests dispatched for PR #${prNumber} by @${context.payload.comment.user.login}. ` + + body: `${label} dispatched for PR #${prNumber} by @${context.payload.comment.user.login}. ` + `Track progress in the [Actions tab](${actionsUrl}).`, }); diff --git a/.github/workflows/min-deps-test-fast.yml b/.github/workflows/main-min-deps.yml similarity index 69% rename from .github/workflows/min-deps-test-fast.yml rename to .github/workflows/main-min-deps.yml index 3365b948c..07eba71b4 100644 --- a/.github/workflows/min-deps-test-fast.yml +++ b/.github/workflows/main-min-deps.yml @@ -1,12 +1,8 @@ # Min-deps variant of main.yml's Tests and Code Checks workflow. Runs # unit tests + a build+verify+smoke-parse job against the committed -# lowest-direct lock. Fires on every merge to main / *.latest (always) -# and on internal PRs (early signal); fork PRs are skipped at the job -# level because they cannot mint the JFrog OIDC token the protected -# runner needs. -# -# Sharded functional coverage lives in the companion min-deps-test-slow.yml -# on a nightly schedule. +# lowest-direct lock. Fires on every push to main / *.latest and on +# every PR (including forks) — dependencies are served from the +# pre-populated cache (see warmDepsCache.yml) when available. name: Tests and Code Checks (Min-Deps) @@ -15,13 +11,24 @@ on: branches: - "main" - "*.latest" + - "releases/*" + paths-ignore: + - "**.MD" + - "**.md" pull_request: + paths-ignore: + - "**.MD" + - "**.md" workflow_dispatch: permissions: id-token: write contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} + cancel-in-progress: true + defaults: run: shell: bash @@ -29,12 +36,10 @@ defaults: jobs: unit: name: unit test / python 3.10 - # Fork PRs cannot mint the JFrog OIDC token; skip them on pull_request. - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: group: databricks-protected-runner-group labels: linux-ubuntu-latest - timeout-minutes: 25 + timeout-minutes: 15 env: UV_FROZEN: "1" @@ -43,7 +48,12 @@ jobs: - name: Check out the repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup JFrog PyPI Proxy + - name: Setup Python Dependencies + id: deps + uses: ./.github/actions/setup-python-deps + + - name: Setup JFrog PyPI Proxy (fallback) + if: steps.deps.outputs.cache-hit != 'true' uses: ./.github/actions/setup-jfrog-pypi - name: Set up Python @@ -64,12 +74,9 @@ jobs: build: name: Build and Verify Packages - # Fork PRs cannot mint the JFrog OIDC token; skip them on pull_request. - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: group: databricks-protected-runner-group labels: linux-ubuntu-latest - timeout-minutes: 15 env: UV_FROZEN: "1" @@ -78,7 +85,12 @@ jobs: - name: Check out the repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup JFrog PyPI Proxy + - name: Setup Python Dependencies + id: deps + uses: ./.github/actions/setup-python-deps + + - name: Setup JFrog PyPI Proxy (fallback) + if: steps.deps.outputs.cache-hit != 'true' uses: ./.github/actions/setup-jfrog-pypi - name: Set up Python diff --git a/.github/workflows/warmDepsCache.yml b/.github/workflows/warmDepsCache.yml index 5525736d8..87d195ae7 100644 --- a/.github/workflows/warmDepsCache.yml +++ b/.github/workflows/warmDepsCache.yml @@ -16,6 +16,7 @@ on: branches: [main] paths: - "uv.lock" + - "requirements.lowest-direct.txt" - "pyproject.toml" - ".pre-commit-config.yaml" schedule: @@ -63,7 +64,7 @@ jobs: git remote add fork "https://github.com/${FORK_REPO}.git" git fetch --depth=1 fork "${FORK_REF}" - git checkout FETCH_HEAD -- uv.lock pyproject.toml .pre-commit-config.yaml + git checkout FETCH_HEAD -- uv.lock requirements.lowest-direct.txt pyproject.toml .pre-commit-config.yaml - name: Setup JFrog PyPI Proxy uses: ./.github/actions/setup-jfrog-pypi @@ -93,6 +94,8 @@ jobs: hatch env create test.py3.12 hatch env create test.py3.13 hatch env create verify + hatch env create min-deps + hatch env create verify-min-deps - name: Warm pre-commit cache run: hatch run pre-commit install-hooks @@ -114,6 +117,8 @@ jobs: hatch run pip download --dest ~/.cache/pip-wheelhouse \ setuptools wheel twine check-wheel-contents \ -r /tmp/runtime-deps.txt + hatch run pip download --dest ~/.cache/pip-wheelhouse \ + --no-deps --require-hashes -r requirements.lowest-direct.txt - name: Build package run: hatch -v build @@ -123,7 +128,7 @@ jobs: shell: bash run: | TIMESTAMP=$(date -u +%Y%m%d%H%M%S) - LOCK_HASH="${{ hashFiles('uv.lock', 'pyproject.toml') }}" + LOCK_HASH="${{ hashFiles('uv.lock', 'requirements.lowest-direct.txt', 'pyproject.toml') }}" echo "python-deps-key=python-deps-${LOCK_HASH}-${TIMESTAMP}" >> "$GITHUB_OUTPUT" PRECOMMIT_HASH="${{ hashFiles('.pre-commit-config.yaml') }}" From 53ac7d5aadc9560f2825b26e54433616f1e6676d Mon Sep 17 00:00:00 2001 From: Shubham Dhal Date: Mon, 25 May 2026 13:00:11 +0530 Subject: [PATCH 2/2] ci: empty commit to re-trigger PR workflows after warm-cache