Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/actions/setup-python-deps/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ runs:
~/.cache/uv
~/.cache/pip
~/.cache/pip-wheelhouse
key: python-deps-${{ hashFiles('uv.lock', 'pyproject.toml') }}-latest
restore-keys: python-deps-${{ hashFiles('uv.lock', 'pyproject.toml') }}-
key: python-deps-${{ hashFiles('uv.lock', 'requirements.lowest-direct.txt', 'pyproject.toml') }}-latest
restore-keys: python-deps-${{ hashFiles('uv.lock', 'requirements.lowest-direct.txt', 'pyproject.toml') }}-

- name: Restore pre-commit cache
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
Expand Down
Original file line number Diff line number Diff line change
@@ -1,15 +1,37 @@
# Min-Deps Integration Tests (slow)
# Integration Tests (Min-Deps) for dbt-databricks
#
# Sharded functional tests against the lowest-direct dependency resolution
# committed in requirements.lowest-direct.txt. Mirrors integration.yml's
# fan-out (skip-if-unchanged on schedule, prepare-shards via LPT historical
# timing, sharded execution per profile, gather-shards verification) but
# routes every pytest invocation through the `min-deps:` hatch env.
#
# Triggering: schedule only.

name: Min-Deps Tests (slow)
# Triggering:
# 1. On a PR (internal OR fork): a maintainer comments `/integration-test min-deps`.
# The integration-trigger workflow validates the comment author and
# dispatches this workflow with the PR number. The run posts a result
# comment back to the PR when the matrix completes.
# 2. Manually from the Actions tab (workflow_dispatch):
# - One PR (e.g. "100") OR comma-separated list ("100,200,300") in pr_numbers.
# - Or a git_ref for ad-hoc testing.
# 3. Nightly on `main` at 19:30 UTC (2h before integration.yml's 21:30 nightly).
# The `prepare` job short-circuits if the current main SHA has already had a
# successful min-deps integration run, emitting an empty targets array so
# the matrix jobs skip cleanly.

name: Integration Tests (Min-Deps)
on:
workflow_dispatch:
inputs:
pr_numbers:
description: "PR number(s) to test — single PR or comma-separated for batch (e.g. '100' or '100,200,300')"
required: false
type: string
git_ref:
description: "Git ref (branch/tag/commit) to test — used only when pr_numbers is empty"
required: false
type: string

schedule:
- cron: "30 19 * * *" # 19:30 UTC, 2h before integration.yml's nightly

Expand All @@ -18,7 +40,7 @@ permissions:
contents: read

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
group: ${{ github.workflow }}-${{ github.event.inputs.pr_numbers || github.event.inputs.git_ref || github.ref }}
cancel-in-progress: true

defaults:
Expand All @@ -43,23 +65,46 @@ jobs:
id: parse
shell: bash
env:
EVENT_NAME: ${{ github.event_name }}
INPUT_PR_NUMBERS: ${{ github.event.inputs.pr_numbers }}
INPUT_GIT_REF: ${{ github.event.inputs.git_ref }}
DEFAULT_REF: ${{ github.ref }}
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
entry() { printf '{"pr":"%s","ref":"%s"}' "$1" "$2"; }
targets="["
# Nightly skip-if-unchanged: if this main SHA already has a green
# min-deps-slow run, emit empty targets so the matrix jobs skip.
already_tested=$(curl -sfS \
-H "Authorization: Bearer $GH_TOKEN" \
-H "Accept: application/vnd.github+json" \
"https://api.github.com/repos/$GITHUB_REPOSITORY/actions/workflows/min-deps-test-slow.yml/runs?branch=main&status=success&head_sha=$GITHUB_SHA" \
| jq -r '.total_count // 0')
if [[ "$already_tested" -gt 0 ]]; then
echo "Nightly skip: main @ $GITHUB_SHA already has $already_tested successful run(s)."
if [[ "$EVENT_NAME" == "schedule" ]]; then
# Nightly skip-if-unchanged: if this main SHA already has a green
# min-deps integration run, emit empty targets so the matrix jobs skip.
already_tested=$(curl -sfS \
-H "Authorization: Bearer $GH_TOKEN" \
-H "Accept: application/vnd.github+json" \
"https://api.github.com/repos/$GITHUB_REPOSITORY/actions/workflows/integration-min-deps.yml/runs?branch=main&status=success&head_sha=$GITHUB_SHA" \
| jq -r '.total_count // 0')
if [[ "$already_tested" -gt 0 ]]; then
echo "Nightly skip: main @ $GITHUB_SHA already has $already_tested successful run(s)."
else
targets+=$(entry "nightly" "$DEFAULT_REF")
fi
elif [[ -n "${INPUT_PR_NUMBERS//[[:space:]]/}" ]]; then
first=1
IFS=',' read -ra prs <<< "$INPUT_PR_NUMBERS"
for pr in "${prs[@]}"; do
pr_trimmed="${pr//[[:space:]]/}"
[[ -z "$pr_trimmed" ]] && continue
if [[ ! "$pr_trimmed" =~ ^[0-9]+$ ]]; then
echo "::error::Invalid PR number '$pr_trimmed' in pr_numbers='$INPUT_PR_NUMBERS' — expected digits, comma-separated."
exit 1
fi
[[ $first -eq 0 ]] && targets+=","
first=0
targets+=$(entry "$pr_trimmed" "refs/pull/$pr_trimmed/head")
done
elif [[ -n "${INPUT_GIT_REF//[[:space:]]/}" ]]; then
targets+=$(entry "manual" "$INPUT_GIT_REF")
else
targets+=$(entry "nightly" "$DEFAULT_REF")
targets+=$(entry "manual" "$DEFAULT_REF")
fi
targets+="]"
echo "targets=$targets" >> "$GITHUB_OUTPUT"
Expand Down Expand Up @@ -97,7 +142,12 @@ jobs:
with:
ref: ${{ matrix.target.ref }}

- name: Setup JFrog PyPI Proxy
- name: Setup Python Dependencies
id: deps
uses: ./.github/actions/setup-python-deps

- name: Setup JFrog PyPI Proxy (fallback)
if: steps.deps.outputs.cache-hit != 'true'
uses: ./.github/actions/setup-jfrog-pypi

- name: Set up python
Expand Down Expand Up @@ -193,7 +243,12 @@ jobs:
with:
ref: ${{ matrix.target.ref }}

- name: Setup JFrog PyPI Proxy
- name: Setup Python Dependencies
id: deps
uses: ./.github/actions/setup-python-deps

- name: Setup JFrog PyPI Proxy (fallback)
if: steps.deps.outputs.cache-hit != 'true'
uses: ./.github/actions/setup-jfrog-pypi

- name: Set up python
Expand Down Expand Up @@ -287,7 +342,12 @@ jobs:
with:
ref: ${{ matrix.target.ref }}

- name: Setup JFrog PyPI Proxy
- name: Setup Python Dependencies
id: deps
uses: ./.github/actions/setup-python-deps

- name: Setup JFrog PyPI Proxy (fallback)
if: steps.deps.outputs.cache-hit != 'true'
uses: ./.github/actions/setup-jfrog-pypi

- name: Set up python
Expand Down Expand Up @@ -379,7 +439,12 @@ jobs:
with:
ref: ${{ matrix.target.ref }}

- name: Setup JFrog PyPI Proxy
- name: Setup Python Dependencies
id: deps
uses: ./.github/actions/setup-python-deps

- name: Setup JFrog PyPI Proxy (fallback)
if: steps.deps.outputs.cache-hit != 'true'
uses: ./.github/actions/setup-jfrog-pypi

- name: Set up python
Expand Down Expand Up @@ -511,3 +576,54 @@ jobs:
echo "::error::One or more shard-verification invariants failed."
exit 1
fi

# Posts a per-job pass/fail summary comment back to the PR when dispatched
# with a single PR number (the slash-command path). Skipped for batch
# dispatches and for schedule / git_ref runs.
report-status:
needs:
- run-uc-cluster-e2e-tests
- run-sqlwarehouse-e2e-tests
- run-cluster-e2e-tests
- gather-shards
if: |
always() &&
github.event_name == 'workflow_dispatch' &&
inputs.pr_numbers != '' &&
!contains(inputs.pr_numbers, ',')
runs-on:
group: databricks-protected-runner-group
labels: linux-ubuntu-latest
permissions:
pull-requests: write
steps:
- name: Post result comment
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
env:
PR_NUMBER: ${{ inputs.pr_numbers }}
UC_RESULT: ${{ needs.run-uc-cluster-e2e-tests.result }}
SQLW_RESULT: ${{ needs.run-sqlwarehouse-e2e-tests.result }}
CLUSTER_RESULT: ${{ needs.run-cluster-e2e-tests.result }}
GATHER_RESULT: ${{ needs.gather-shards.result }}
with:
script: |
const ICONS = { success: ':white_check_mark:', skipped: ':fast_forward:' };
const results = {
'UC cluster': process.env.UC_RESULT,
'SQL warehouse': process.env.SQLW_RESULT,
'All-purpose cluster': process.env.CLUSTER_RESULT,
'Shard coverage': process.env.GATHER_RESULT,
};
const line = Object.entries(results)
.map(([name, r]) => `${name} ${ICONS[r] || ':x:'} ${r}`)
.join(' · ');
const runUrl =
`https://github.com/${context.repo.owner}/${context.repo.repo}` +
`/actions/runs/${context.runId}`;
const prNumber = process.env.PR_NUMBER.trim();
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: parseInt(prNumber, 10),
body: `Min-deps integration results for PR #${prNumber} — ${line}\n\n[Run details](${runUrl}).`,
});
27 changes: 18 additions & 9 deletions .github/workflows/integration-trigger.yml
Original file line number Diff line number Diff line change
@@ -1,9 +1,15 @@
# Integration Test Trigger
#
# Listens for `/integration-test` PR comments and dispatches the integration
# test workflow against the PR head. Works uniformly for internal and forked
# PRs because the dispatch runs in the main repo context (which has access to
# the Databricks secrets).
# Listens for `/integration-test` PR comments and dispatches the appropriate
# integration test workflow against the PR head. Works uniformly for internal
# and forked PRs because the dispatch runs in the main repo context (which
# has access to the Databricks secrets).
#
# Routing on the first whitespace-separated arg after `/integration-test`:
# /integration-test → integration.yml (latest deps)
# /integration-test min-deps → integration-min-deps.yml (lowest-direct lock)
# /integration-test <anything else> → integration.yml (trailing text
# is free-form context, e.g. retry notes)
#
# Authorization: only users whose author_association is OWNER, MEMBER, or
# COLLABORATOR can trigger a run. Anyone else gets a reply comment explaining
Expand All @@ -22,8 +28,7 @@ permissions:
jobs:
dispatch:
# Exact command match, or the command followed by whitespace (so maintainers
# can add context like `/integration-test please retry the sqlw flake`) —
# never matches `/integration-test-foo`.
# can add context or modifiers) — never matches `/integration-test-foo`.
if: |
github.event.issue.pull_request &&
(github.event.comment.body == '/integration-test' || startsWith(github.event.comment.body, '/integration-test ')) &&
Expand All @@ -47,22 +52,26 @@ jobs:
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
with:
script: |
const rest = context.payload.comment.body.slice('/integration-test'.length).trim();
const isMinDeps = rest === 'min-deps' || rest.startsWith('min-deps ');
const workflowId = isMinDeps ? 'integration-min-deps.yml' : 'integration.yml';
const label = isMinDeps ? 'Min-deps integration tests' : 'Integration tests';
const prNumber = String(context.payload.issue.number);
await github.rest.actions.createWorkflowDispatch({
owner: context.repo.owner,
repo: context.repo.repo,
workflow_id: 'integration.yml',
workflow_id: workflowId,
ref: 'main',
inputs: { pr_numbers: prNumber },
});
const actionsUrl =
`https://github.com/${context.repo.owner}/${context.repo.repo}` +
`/actions/workflows/integration.yml`;
`/actions/workflows/${workflowId}`;
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.payload.issue.number,
body: `Integration tests dispatched for PR #${prNumber} by @${context.payload.comment.user.login}. ` +
body: `${label} dispatched for PR #${prNumber} by @${context.payload.comment.user.login}. ` +
`Track progress in the [Actions tab](${actionsUrl}).`,
});

Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,8 @@
# Min-deps variant of main.yml's Tests and Code Checks workflow. Runs
# unit tests + a build+verify+smoke-parse job against the committed
# lowest-direct lock. Fires on every merge to main / *.latest (always)
# and on internal PRs (early signal); fork PRs are skipped at the job
# level because they cannot mint the JFrog OIDC token the protected
# runner needs.
#
# Sharded functional coverage lives in the companion min-deps-test-slow.yml
# on a nightly schedule.
# lowest-direct lock. Fires on every push to main / *.latest and on
# every PR (including forks) — dependencies are served from the
# pre-populated cache (see warmDepsCache.yml) when available.

name: Tests and Code Checks (Min-Deps)

Expand All @@ -15,26 +11,35 @@ on:
branches:
- "main"
- "*.latest"
- "releases/*"
paths-ignore:
- "**.MD"
- "**.md"
pull_request:
paths-ignore:
- "**.MD"
- "**.md"
workflow_dispatch:

permissions:
id-token: write
contents: read

concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}
cancel-in-progress: true

defaults:
run:
shell: bash

jobs:
unit:
name: unit test / python 3.10
# Fork PRs cannot mint the JFrog OIDC token; skip them on pull_request.
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
runs-on:
group: databricks-protected-runner-group
labels: linux-ubuntu-latest
timeout-minutes: 25
timeout-minutes: 15

env:
UV_FROZEN: "1"
Expand All @@ -43,7 +48,12 @@ jobs:
- name: Check out the repository
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4

- name: Setup JFrog PyPI Proxy
- name: Setup Python Dependencies
id: deps
uses: ./.github/actions/setup-python-deps

- name: Setup JFrog PyPI Proxy (fallback)
if: steps.deps.outputs.cache-hit != 'true'
uses: ./.github/actions/setup-jfrog-pypi

- name: Set up Python
Expand All @@ -64,12 +74,9 @@ jobs:

build:
name: Build and Verify Packages
# Fork PRs cannot mint the JFrog OIDC token; skip them on pull_request.
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
runs-on:
group: databricks-protected-runner-group
labels: linux-ubuntu-latest
timeout-minutes: 15

env:
UV_FROZEN: "1"
Expand All @@ -78,7 +85,12 @@ jobs:
- name: Check out the repository
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4

- name: Setup JFrog PyPI Proxy
- name: Setup Python Dependencies
id: deps
uses: ./.github/actions/setup-python-deps

- name: Setup JFrog PyPI Proxy (fallback)
if: steps.deps.outputs.cache-hit != 'true'
uses: ./.github/actions/setup-jfrog-pypi

- name: Set up Python
Expand Down
Loading
Loading