From 508a989f79e2dacb1c68b0b68c3eadc3a7f214f2 Mon Sep 17 00:00:00 2001 From: beinan Date: Wed, 18 Feb 2026 23:21:06 +0000 Subject: [PATCH 1/7] ci: enable Rust build artifact caching in Python tests Fix Rust cache configuration to properly cache compiled artifacts. This should significantly reduce the maturin develop step time in CI by reusing cached builds when Rust code hasn't changed. Changes: - Set correct workspace path to crates/lance-graph-python - Remove cache-targets: false to enable artifact caching (defaults to true) Co-Authored-By: Claude Opus 4.6 --- .github/workflows/python-test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 97f4caab..1bee8ce3 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -36,8 +36,7 @@ jobs: uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: Swatinem/rust-cache@v2 with: - workspaces: python - cache-targets: false + workspaces: crates/lance-graph-python - name: Install dependencies run: | sudo apt update From c5a648d76708189d8a2c0d2dc5df2e82d170273a Mon Sep 17 00:00:00 2001 From: beinan Date: Thu, 19 Feb 2026 00:40:44 +0000 Subject: [PATCH 2/7] ci: enable shared Rust dependency cache across all workflows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Configure all workflows to share a single Rust dependency cache using shared-key. This allows different workflows to reuse compiled artifacts for dependencies like Arrow, DataFusion, Tokio, and aws-lc-sys. Changes: - Add shared-key: "lance-graph-deps" to all rust-cache configurations - Include both crates/lance-graph and crates/lance-graph-python workspaces - Remove cache-targets: false to enable target directory caching - Apply to: python-test.yml, build.yml, rust-test.yml, style.yml Expected impact: - First workflow run compiles dependencies and populates shared cache - Subsequent workflows (even different ones) reuse cached dependencies - Massive time savings: 5+ min build → <1 min for unchanged dependencies Co-Authored-By: Claude Opus 4.6 --- .github/workflows/build.yml | 6 ++++-- .github/workflows/python-test.yml | 5 ++++- .github/workflows/rust-test.yml | 12 ++++++++---- .github/workflows/style.yml | 5 ++++- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2cc6bdc7..ff70e1b4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,8 +36,10 @@ jobs: rustup default ${{ matrix.toolchain }} - uses: Swatinem/rust-cache@v2 with: - workspaces: crates/lance-graph - cache-targets: false + shared-key: "lance-graph-deps" + workspaces: | + crates/lance-graph + crates/lance-graph-python - name: Install dependencies run: | sudo apt update diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 1bee8ce3..a106b621 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -36,7 +36,10 @@ jobs: uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: Swatinem/rust-cache@v2 with: - workspaces: crates/lance-graph-python + shared-key: "lance-graph-deps" + workspaces: | + crates/lance-graph + crates/lance-graph-python - name: Install dependencies run: | sudo apt update diff --git a/.github/workflows/rust-test.yml b/.github/workflows/rust-test.yml index 22bcb7fa..3c2b19f0 100644 --- a/.github/workflows/rust-test.yml +++ b/.github/workflows/rust-test.yml @@ -37,8 +37,10 @@ jobs: rustup default ${{ matrix.toolchain }} - uses: Swatinem/rust-cache@v2 with: - workspaces: crates/lance-graph - cache-targets: false + shared-key: "lance-graph-deps" + workspaces: | + crates/lance-graph + crates/lance-graph-python - name: Install dependencies run: | sudo apt update @@ -61,8 +63,10 @@ jobs: rustup default stable - uses: Swatinem/rust-cache@v2 with: - workspaces: crates/lance-graph - cache-targets: false + shared-key: "lance-graph-deps" + workspaces: | + crates/lance-graph + crates/lance-graph-python - name: Install dependencies run: | sudo apt update diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index cb7878fb..d240ff78 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -37,7 +37,10 @@ jobs: - uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 with: - workspaces: crates/lance-graph + shared-key: "lance-graph-deps" + workspaces: | + crates/lance-graph + crates/lance-graph-python - name: Install dependencies run: | sudo apt update From 224b35bbe25a32d137fb4f24e965661eb0e3deff Mon Sep 17 00:00:00 2001 From: beinan Date: Thu, 19 Feb 2026 05:55:11 +0000 Subject: [PATCH 3/7] test: verify shared cache performance This trivial change will test if the shared Rust cache is working. Expected: Build time should drop from 23min to <2min. --- .github/workflows/python-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index a106b621..097339f2 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -104,3 +104,4 @@ jobs: run: | source .venv/bin/activate pyright +# Cache test - verifying shared cache works From 0df9693cff71c6c382864e74a97c7307701f55f9 Mon Sep 17 00:00:00 2001 From: beinan Date: Thu, 19 Feb 2026 07:51:07 +0000 Subject: [PATCH 4/7] ci: enable incremental compilation for effective caching Root cause: CARGO_INCREMENTAL=0 was preventing cargo from reusing any cached compilation artifacts, making rust-cache ineffective. Setting CARGO_INCREMENTAL=1 allows cargo to: - Reuse compiled .rlib files from cache - Skip recompiling unchanged dependencies - Dramatically reduce build times (5min -> <1min expected) Applied to: python-test.yml, build.yml, rust-test.yml Co-Authored-By: Claude Opus 4.6 --- .github/workflows/build.yml | 2 +- .github/workflows/python-test.yml | 2 +- .github/workflows/rust-test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ff70e1b4..a015ed9f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ env: CARGO_TERM_COLOR: always RUSTFLAGS: "-C debuginfo=1" RUST_BACKTRACE: "1" - CARGO_INCREMENTAL: "0" + CARGO_INCREMENTAL: "1" jobs: linux-build: diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 097339f2..3eb7a98f 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -18,6 +18,7 @@ concurrency: env: CARGO_TERM_COLOR: always RUSTFLAGS: "-C debuginfo=1" + CARGO_INCREMENTAL: "1" jobs: test: @@ -104,4 +105,3 @@ jobs: run: | source .venv/bin/activate pyright -# Cache test - verifying shared cache works diff --git a/.github/workflows/rust-test.yml b/.github/workflows/rust-test.yml index 3c2b19f0..41477900 100644 --- a/.github/workflows/rust-test.yml +++ b/.github/workflows/rust-test.yml @@ -18,7 +18,7 @@ env: CARGO_TERM_COLOR: always RUSTFLAGS: "-C debuginfo=1" RUST_BACKTRACE: "1" - CARGO_INCREMENTAL: "0" + CARGO_INCREMENTAL: "1" jobs: test: From 58444b78d0179368cb00e8cf9d5da6a4a5d52a00 Mon Sep 17 00:00:00 2001 From: beinan Date: Tue, 24 Feb 2026 00:21:30 +0000 Subject: [PATCH 5/7] ci: fix python makefile for release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Skip editable install during dependency setup to avoid building Rust extension twice. The workflow was: 1. uv pip install -e .[tests] → builds Rust (18 min) 2. maturin develop → builds Rust again (5 min) Now only maturin develop builds once, reducing CI time from 24min to ~6min. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/python-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 3eb7a98f..3aa776bb 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -53,7 +53,8 @@ jobs: uv venv source .venv/bin/activate uv pip install maturin[patchelf] - uv pip install -e .[tests] + # Install test dependencies only (skip editable install to avoid double-building Rust) + uv pip install pytest pyarrow pandas ruff - name: Build Python extension working-directory: python run: | From bc32fff874a4ec8e2fd9fc19d7b8df005edbd669 Mon Sep 17 00:00:00 2001 From: beinan Date: Tue, 24 Feb 2026 02:10:40 +0000 Subject: [PATCH 6/7] ci: remove nightly from Rust test matrix Remove nightly toolchain testing to prevent upstream dependency breakage from blocking PRs. This matches the upstream lance project's approach, which only uses nightly for specific jobs (coverage). Current issue: shellexpand v3.1.1 fails to compile on nightly Rust with type mismatch errors. Since this is a transitive dependency, we cannot fix it directly. Benefits: - Prevents nightly breakage from blocking PRs - Follows upstream lance pattern - Stable toolchain is the primary target anyway Co-Authored-By: Claude Opus 4.6 --- .github/workflows/rust-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/rust-test.yml b/.github/workflows/rust-test.yml index 41477900..473e041f 100644 --- a/.github/workflows/rust-test.yml +++ b/.github/workflows/rust-test.yml @@ -28,7 +28,6 @@ jobs: matrix: toolchain: - stable - - nightly steps: - uses: actions/checkout@v4 - name: Setup rust toolchain From 77eb44f6ac573f22afe9fa52a26e9293a9df1baa Mon Sep 17 00:00:00 2001 From: beinan Date: Tue, 24 Feb 2026 03:25:51 +0000 Subject: [PATCH 7/7] ci: revert CARGO_INCREMENTAL to 0 Set CARGO_INCREMENTAL=0 explicitly since rust-cache overrides it to 0 anyway. This makes the intention clear and matches the CI convention. Incremental compilation artifacts are not useful in CI because: - They're workspace-specific (only help with unchanged code) - rust-cache only caches dependencies, not workspace code - Workspace code changes every commit - Creating incremental artifacts wastes time/space Co-Authored-By: Claude Opus 4.6 --- .github/workflows/build.yml | 2 +- .github/workflows/python-test.yml | 2 +- .github/workflows/rust-test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a015ed9f..ff70e1b4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ env: CARGO_TERM_COLOR: always RUSTFLAGS: "-C debuginfo=1" RUST_BACKTRACE: "1" - CARGO_INCREMENTAL: "1" + CARGO_INCREMENTAL: "0" jobs: linux-build: diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 3aa776bb..ee5c92d8 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -18,7 +18,7 @@ concurrency: env: CARGO_TERM_COLOR: always RUSTFLAGS: "-C debuginfo=1" - CARGO_INCREMENTAL: "1" + CARGO_INCREMENTAL: "0" jobs: test: diff --git a/.github/workflows/rust-test.yml b/.github/workflows/rust-test.yml index 473e041f..fc02408b 100644 --- a/.github/workflows/rust-test.yml +++ b/.github/workflows/rust-test.yml @@ -18,7 +18,7 @@ env: CARGO_TERM_COLOR: always RUSTFLAGS: "-C debuginfo=1" RUST_BACKTRACE: "1" - CARGO_INCREMENTAL: "1" + CARGO_INCREMENTAL: "0" jobs: test: