From 75bebe9da93b5368403830b58fbc70558d3d2a22 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Mon, 8 Dec 2025 18:37:15 +0000 Subject: [PATCH 1/8] Fix conflicts Signed-off-by: Adam Gutglick --- Cargo.lock | 2254 ++++++++++++++------ Cargo.toml | 60 +- bench-vortex/src/public_bi.rs | 2 +- vortex-datafusion/src/convert/scalars.rs | 126 +- vortex-datafusion/src/persistent/mod.rs | 5 +- vortex-datafusion/src/persistent/opener.rs | 177 +- vortex-datafusion/src/persistent/source.rs | 20 +- vortex-dtype/src/arrow.rs | 11 +- 8 files changed, 1780 insertions(+), 875 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6dc2fef6c90..a9ccded7179 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -35,9 +35,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] @@ -104,22 +104,22 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "anstyle-wincon" -version = "3.0.10" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -176,19 +176,40 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-csv", - "arrow-data", - "arrow-ipc", - "arrow-json", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-arith 56.2.0", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-csv 56.2.0", + "arrow-data 56.2.0", + "arrow-ipc 56.2.0", + "arrow-json 56.2.0", + "arrow-ord 56.2.0", + "arrow-row 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "arrow-string 56.2.0", +] + +[[package]] +name = "arrow" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4df8bb5b0bd64c0b9bc61317fcc480bad0f00e56d3bc32c69a4c8dada4786bae" +dependencies = [ + "arrow-arith 57.0.0", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-cast 57.0.0", + "arrow-csv 57.0.0", + "arrow-data 57.0.0", + "arrow-ipc 57.0.0", + "arrow-json 57.0.0", + "arrow-ord 57.0.0", + "arrow-row 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", + "arrow-string 57.0.0", ] [[package]] @@ -197,14 +218,28 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "chrono", "num", ] +[[package]] +name = "arrow-arith" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a640186d3bd30a24cb42264c2dafb30e236a6f50d510e56d40b708c9582491" +dependencies = [ + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "chrono", + "num-traits", +] + [[package]] name = "arrow-array" version = "56.2.0" @@ -212,9 +247,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" dependencies = [ "ahash", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "chrono", "chrono-tz", "half", @@ -222,6 +257,25 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-array" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219fe420e6800979744c8393b687afb0252b3f8a89b91027d27887b72aa36d31" +dependencies = [ + "ahash", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.16.1", + "num-complex", + "num-integer", + "num-traits", +] + [[package]] name = "arrow-buffer" version = "56.2.0" @@ -233,17 +287,29 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-buffer" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76885a2697a7edf6b59577f568b456afc94ce0e2edc15b784ce3685b6c3c5c27" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + [[package]] name = "arrow-cast" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "atoi", "base64", "chrono", @@ -254,15 +320,51 @@ dependencies = [ "ryu", ] +[[package]] +name = "arrow-cast" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9ebb4c987e6b3b236fb4a14b20b34835abfdd80acead3ccf1f9bf399e1f168" +dependencies = [ + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + [[package]] name = "arrow-csv" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa9bf02705b5cf762b6f764c65f04ae9082c7cfc4e96e0c33548ee3f67012eb" dependencies = [ - "arrow-array", - "arrow-cast", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-cast 56.2.0", + "arrow-schema 56.2.0", + "chrono", + "csv", + "csv-core", + "regex", +] + +[[package]] +name = "arrow-csv" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92386159c8d4bce96f8bd396b0642a0d544d471bdc2ef34d631aec80db40a09c" +dependencies = [ + "arrow-array 57.0.0", + "arrow-cast 57.0.0", + "arrow-schema 57.0.0", "chrono", "csv", "csv-core", @@ -275,39 +377,67 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 56.2.0", + "arrow-schema 56.2.0", "half", "num", ] +[[package]] +name = "arrow-data" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727681b95de313b600eddc2a37e736dcb21980a40f640314dcf360e2f36bc89b" +dependencies = [ + "arrow-buffer 57.0.0", + "arrow-schema 57.0.0", + "half", + "num-integer", + "num-traits", +] + [[package]] name = "arrow-ipc" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d3594dcddccc7f20fd069bc8e9828ce37220372680ff638c5e00dea427d88f5" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "flatbuffers", "lz4_flex", "zstd", ] +[[package]] +name = "arrow-ipc" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9ba92e3de170295c98a84e5af22e2b037f0c7b32449445e6c493b5fca27f27" +dependencies = [ + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", + "flatbuffers", + "lz4_flex", +] + [[package]] name = "arrow-json" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88cf36502b64a127dc659e3b305f1d993a544eab0d48cce704424e62074dc04b" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "chrono", "half", "indexmap", @@ -319,17 +449,54 @@ dependencies = [ "simdutf8", ] +[[package]] +name = "arrow-json" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b969b4a421ae83828591c6bf5450bd52e6d489584142845ad6a861f42fe35df8" +dependencies = [ + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-cast 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "chrono", + "half", + "indexmap", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + [[package]] name = "arrow-ord" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", +] + +[[package]] +name = "arrow-ord" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "141c05298b21d03e88062317a1f1a73f5ba7b6eb041b350015b1cd6aabc0519b" +dependencies = [ + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", ] [[package]] @@ -338,10 +505,23 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "half", +] + +[[package]] +name = "arrow-row" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f3c06a6abad6164508ed283c7a02151515cef3de4b4ff2cebbcaeb85533db2" +dependencies = [ + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", "half", ] @@ -356,6 +536,17 @@ dependencies = [ "serde_json", ] +[[package]] +name = "arrow-schema" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cfa7a03d1eee2a4d061476e1840ad5c9867a544ca6c4c59256496af5d0a8be5" +dependencies = [ + "bitflags", + "serde_core", + "serde_json", +] + [[package]] name = "arrow-select" version = "56.2.0" @@ -363,30 +554,61 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "num", ] +[[package]] +name = "arrow-select" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bafa595babaad59f2455f4957d0f26448fb472722c186739f4fac0823a1bdb47" +dependencies = [ + "ahash", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "num-traits", +] + [[package]] name = "arrow-string" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "memchr", "num", "regex", "regex-syntax", ] +[[package]] +name = "arrow-string" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f46457dbbb99f2650ff3ac23e46a929e0ab81db809b02aa5511c258348bef2" +dependencies = [ + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + [[package]] name = "async-channel" version = "2.5.0" @@ -414,9 +636,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.32" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a89bce6054c720275ac2432fbba080a66a2106a44a1b804553930ca6909f4e0" +checksum = "0e86f6d3dc9dc4352edeea6b8e499e13e3f5dc3b964d7ca5fd411415a3498473" dependencies = [ "compression-codecs", "compression-core", @@ -516,7 +738,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -556,7 +778,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -573,7 +795,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -608,9 +830,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-config" -version = "1.8.8" +version = "1.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37cf2b6af2a95a20e266782b4f76f1a5e12bf412a9db2de9c1e9123b9d8c0ad8" +checksum = "a0149602eeaf915158e14029ba0c78dedb8c08d554b024d54c8f239aab46511d" dependencies = [ "aws-credential-types", "aws-runtime", @@ -638,9 +860,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.8" +version = "1.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf26925f4a5b59eb76722b63c2892b1d70d06fa053c72e4a100ec308c1d47bc" +checksum = "b01c9521fa01558f750d183c8c68c81b0155b9d193a4ba7f84c36bd1b6d04a06" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -650,9 +872,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.14.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b8ff6c09cd57b16da53641caa860168b88c172a5ee163b0288d3d6eea12786" +checksum = "6b5ce75405893cd713f9ab8e297d8e438f624dde7d706108285f7e17a25a180f" dependencies = [ "aws-lc-sys", "zeroize", @@ -660,11 +882,10 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.31.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e44d16778acaf6a9ec9899b92cebd65580b83f685446bf2e1f5d3d732f99dcd" +checksum = "179c3777a8b5e70e90ea426114ffc565b2c1a9f82f6c4a0c5a34aa6ef5e781b6" dependencies = [ - "bindgen", "cc", "cmake", "dunce", @@ -673,9 +894,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.12" +version = "1.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa006bb32360ed90ac51203feafb9d02e3d21046e1fd3a450a404b90ea73e5d" +checksum = "7ce527fb7e53ba9626fc47824f25e256250556c40d8f81d27dd92aa38239d632" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -697,9 +918,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.86.0" +version = "1.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a0abbfab841446cce6e87af853a3ba2cc1bc9afcd3f3550dd556c43d434c86d" +checksum = "4f18e53542c522459e757f81e274783a78f8c81acdfc8d1522ee8a18b5fb1c66" dependencies = [ "aws-credential-types", "aws-runtime", @@ -719,9 +940,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.88.0" +version = "1.92.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a68d675582afea0e94d38b6ca9c5aaae4ca14f1d36faa6edb19b42e687e70d7" +checksum = "532f4d866012ffa724a4385c82e8dd0e59f0ca0e600f3f22d4c03b6824b34e4a" dependencies = [ "aws-credential-types", "aws-runtime", @@ -741,9 +962,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.88.0" +version = "1.94.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d30990923f4f675523c51eb1c0dec9b752fb267b36a61e83cbc219c9d86da715" +checksum = "1be6fbbfa1a57724788853a623378223fe828fc4c09b146c992f0c95b6256174" dependencies = [ "aws-credential-types", "aws-runtime", @@ -764,9 +985,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.3.5" +version = "1.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bffc03068fbb9c8dd5ce1c6fb240678a5cffb86fb2b7b1985c999c4b83c8df68" +checksum = "c35452ec3f001e1f2f6db107b6373f1f48f05ec63ba2c5c9fa91f07dad32af11" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -797,15 +1018,16 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.62.4" +version = "0.62.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3feafd437c763db26aa04e0cc7591185d0961e64c61885bece0fb9d50ceac671" +checksum = "445d5d720c99eed0b4aa674ed00d835d9b1427dd73e04adaf2f94c6b2d6f9fca" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "bytes-utils", "futures-core", + "futures-util", "http 0.2.12", "http 1.3.1", "http-body 0.4.6", @@ -817,9 +1039,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1053b5e587e6fa40ce5a79ea27957b04ba660baa02b28b7436f64850152234f1" +checksum = "623254723e8dfd535f566ee7b2381645f8981da086b5c4aa26c0c41582bb1d2c" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -841,9 +1063,9 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.61.6" +version = "0.61.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff418fc8ec5cadf8173b10125f05c2e7e1d46771406187b2c878557d4503390" +checksum = "2db31f727935fc63c6eeae8b37b438847639ec330a9161ece694efba257e0c54" dependencies = [ "aws-smithy-types", ] @@ -869,9 +1091,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.9.3" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ab99739082da5347660c556689256438defae3bcefd66c52b095905730e404" +checksum = "0bbe9d018d646b96c7be063dd07987849862b0e6d07c778aad7d93d1be6c1ef0" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -893,9 +1115,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3683c5b152d2ad753607179ed71988e8cfd52964443b4f74fd8e552d0bbfeb46" +checksum = "ec7204f9fd94749a7c53b26da1b961b4ac36bf070ef1e0b94bb09f79d4f6c193" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -910,9 +1132,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.3.3" +version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f5b3a7486f6690ba25952cabf1e7d75e34d69eaff5081904a47bc79074d6457" +checksum = "25f535879a207fce0db74b679cfc3e91a3159c8144d717d55f5832aea9eef46e" dependencies = [ "base64-simd", "bytes", @@ -933,18 +1155,18 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.11" +version = "0.60.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9c34127e8c624bc2999f3b657e749c1393bedc9cd97b92a804db8ced4d2e163" +checksum = "eab77cdd036b11056d2a30a7af7b775789fb024bf216acc13884c6c97752ae56" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "1.3.9" +version = "1.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2fd329bf0e901ff3f60425691410c69094dc2a1f34b331f37bfc4e9ac1565a1" +checksum = "d79fb68e3d7fe5d4833ea34dc87d2e97d26d3086cb3da660bb6b1f76d98680b6" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -956,9 +1178,9 @@ dependencies = [ [[package]] name = "backon" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "592277618714fbcecda9a02ba7a8781f319d26532a88553bbacc77ba5d2b3a8d" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" dependencies = [ "fastrand", "gloo-timers", @@ -992,17 +1214,17 @@ name = "bench-vortex" version = "0.1.0" dependencies = [ "anyhow", - "arrow-array", - "arrow-cast", - "arrow-schema", - "arrow-select", + "arrow-array 57.0.0", + "arrow-cast 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", "async-trait", "bytes", "bzip2", "clap", - "datafusion", - "datafusion-common", - "datafusion-physical-plan", + "datafusion 51.0.0", + "datafusion-common 51.0.0", + "datafusion-physical-plan 51.0.0", "dirs", "erased-serde", "futures", @@ -1059,9 +1281,9 @@ checksum = "92fde17f91e7ba10b2a07f8dff29530b77144894bc6ae850fbc66e1276af0d28" [[package]] name = "bigdecimal" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a22f228ab7a1b23027ccc6c350b72868017af7ea8356fbdf19f8d991c690013" +checksum = "560f42649de9fa436b73517378a147ec21f6c997a546581df4b4b31677828934" dependencies = [ "autocfg", "libm", @@ -1087,7 +1309,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -1098,9 +1320,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.9.4" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" [[package]] name = "bitpacking" @@ -1198,7 +1420,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -1224,9 +1446,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" dependencies = [ "memchr", "regex-automata", @@ -1326,16 +1548,16 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.106", + "syn 2.0.110", "tempfile", "toml", ] [[package]] name = "cc" -version = "1.2.49" +version = "1.2.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215" +checksum = "cd405d82c84ff7f35739f175f67d8b9fb7687a0e84ccdc78bd3568839827cf07" dependencies = [ "find-msvc-tools", "jobserver", @@ -1453,14 +1675,14 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] name = "clap_lex" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" [[package]] name = "cmake" @@ -1524,7 +1746,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -1595,9 +1817,9 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.31" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8a506ec4b81c460798f572caead636d57d3d7e940f998160f52bd254bf2d23" +checksum = "302266479cb963552d11bd042013a58ef1adc56768016c8b82b4199488f2d4ad" dependencies = [ "compression-core", "flate2", @@ -1606,9 +1828,9 @@ dependencies = [ [[package]] name = "compression-core" -version = "0.4.29" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e47641d3deaf41fb1538ac1f54735925e275eaf3bf4d55c81b137fba797e5cbb" +checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" [[package]] name = "concurrent-queue" @@ -1879,9 +2101,9 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", "typenum", @@ -1889,21 +2111,21 @@ dependencies = [ [[package]] name = "csv" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" dependencies = [ "csv-core", "itoa", "ryu", - "serde", + "serde_core", ] [[package]] name = "csv-core" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" dependencies = [ "memchr", ] @@ -1945,7 +2167,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -1959,7 +2181,7 @@ dependencies = [ "indexmap", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -1977,7 +2199,7 @@ dependencies = [ "indexmap", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -2011,7 +2233,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -2025,7 +2247,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -2036,7 +2258,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -2047,7 +2269,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -2070,36 +2292,85 @@ version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af15bb3c6ffa33011ef579f6b0bcbe7c26584688bd6c994f548e44df67f011a" dependencies = [ - "arrow", - "arrow-ipc", - "arrow-schema", + "arrow 56.2.0", + "arrow-ipc 56.2.0", + "arrow-schema 56.2.0", "async-trait", "bytes", "chrono", - "datafusion-catalog", - "datafusion-catalog-listing", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-datasource-csv", - "datafusion-datasource-json", + "datafusion-catalog 50.3.0", + "datafusion-catalog-listing 50.3.0", + "datafusion-common 50.3.0", + "datafusion-common-runtime 50.3.0", + "datafusion-datasource 50.3.0", + "datafusion-datasource-csv 50.3.0", + "datafusion-datasource-json 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-expr-common 50.3.0", + "datafusion-functions 50.3.0", + "datafusion-functions-aggregate 50.3.0", + "datafusion-functions-nested 50.3.0", + "datafusion-functions-table 50.3.0", + "datafusion-functions-window 50.3.0", + "datafusion-optimizer 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-adapter 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "datafusion-physical-optimizer 50.3.0", + "datafusion-physical-plan 50.3.0", + "datafusion-session 50.3.0", + "datafusion-sql 50.3.0", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "rand 0.9.2", + "regex", + "sqlparser 0.58.0", + "tempfile", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "datafusion" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ba7cb113e9c0bedf9e9765926031e132fa05a1b09ba6e93a6d1a4d7044457b8" +dependencies = [ + "arrow 57.0.0", + "arrow-schema 57.0.0", + "async-trait", + "bytes", + "chrono", + "datafusion-catalog 51.0.0", + "datafusion-catalog-listing 51.0.0", + "datafusion-common 51.0.0", + "datafusion-common-runtime 51.0.0", + "datafusion-datasource 51.0.0", + "datafusion-datasource-arrow", + "datafusion-datasource-csv 51.0.0", + "datafusion-datasource-json 51.0.0", "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", - "datafusion-functions-table", - "datafusion-functions-window", - "datafusion-optimizer", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-session", - "datafusion-sql", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-expr-common 51.0.0", + "datafusion-functions 51.0.0", + "datafusion-functions-aggregate 51.0.0", + "datafusion-functions-nested 51.0.0", + "datafusion-functions-table 51.0.0", + "datafusion-functions-window 51.0.0", + "datafusion-optimizer 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-adapter 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-optimizer 51.0.0", + "datafusion-physical-plan 51.0.0", + "datafusion-session 51.0.0", + "datafusion-sql 51.0.0", "futures", "itertools 0.14.0", "log", @@ -2108,7 +2379,8 @@ dependencies = [ "parquet", "rand 0.9.2", "regex", - "sqlparser", + "rstest", + "sqlparser 0.59.0", "tempfile", "tokio", "url", @@ -2121,18 +2393,43 @@ version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "187622262ad8f7d16d3be9202b4c1e0116f1c9aa387e5074245538b755261621" dependencies = [ - "arrow", + "arrow 56.2.0", + "async-trait", + "dashmap", + "datafusion-common 50.3.0", + "datafusion-common-runtime 50.3.0", + "datafusion-datasource 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-plan 50.3.0", + "datafusion-session 50.3.0", + "datafusion-sql 50.3.0", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-catalog" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66a3a799f914a59b1ea343906a0486f17061f39509af74e874a866428951130d" +dependencies = [ + "arrow 57.0.0", "async-trait", "dashmap", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-session", - "datafusion-sql", + "datafusion-common 51.0.0", + "datafusion-common-runtime 51.0.0", + "datafusion-datasource 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-plan 51.0.0", + "datafusion-session 51.0.0", "futures", "itertools 0.14.0", "log", @@ -2147,18 +2444,42 @@ version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9657314f0a32efd0382b9a46fdeb2d233273ece64baa68a7c45f5a192daf0f83" dependencies = [ - "arrow", + "arrow 56.2.0", + "async-trait", + "datafusion-catalog 50.3.0", + "datafusion-common 50.3.0", + "datafusion-datasource 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "datafusion-physical-plan 50.3.0", + "datafusion-session 50.3.0", + "futures", + "log", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db1b113c80d7a0febcd901476a57aef378e717c54517a163ed51417d87621b0" +dependencies = [ + "arrow 57.0.0", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-catalog 51.0.0", + "datafusion-common 51.0.0", + "datafusion-datasource 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-adapter 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-plan 51.0.0", "futures", + "itertools 0.14.0", "log", "object_store", "tokio", @@ -2171,8 +2492,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a83760d9a13122d025fbdb1d5d5aaf93dd9ada5e90ea229add92aa30898b2d1" dependencies = [ "ahash", - "arrow", - "arrow-ipc", + "arrow 56.2.0", + "arrow-ipc 56.2.0", "base64", "chrono", "half", @@ -2181,9 +2502,31 @@ dependencies = [ "libc", "log", "object_store", + "paste", + "sqlparser 0.58.0", + "tokio", + "web-time", +] + +[[package]] +name = "datafusion-common" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c10f7659e96127d25e8366be7c8be4109595d6a2c3eac70421f380a7006a1b0" +dependencies = [ + "ahash", + "arrow 57.0.0", + "arrow-ipc 57.0.0", + "chrono", + "half", + "hashbrown 0.14.5", + "indexmap", + "libc", + "log", + "object_store", "parquet", "paste", - "sqlparser", + "sqlparser 0.59.0", "tokio", "web-time", ] @@ -2199,56 +2542,141 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-common-runtime" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b92065bbc6532c6651e2f7dd30b55cba0c7a14f860c7e1d15f165c41a1868d95" +dependencies = [ + "futures", + "log", + "tokio", +] + [[package]] name = "datafusion-datasource" version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7256c9cb27a78709dd42d0c80f0178494637209cac6e29d5c93edd09b6721b86" dependencies = [ - "arrow", + "arrow 56.2.0", "async-trait", "bytes", "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 50.3.0", + "datafusion-common-runtime 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-adapter 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "datafusion-physical-plan 50.3.0", + "datafusion-session 50.3.0", + "futures", + "glob", + "itertools 0.14.0", + "log", + "object_store", + "rand 0.9.2", + "tokio", + "url", +] + +[[package]] +name = "datafusion-datasource" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fde13794244bc7581cd82f6fff217068ed79cdc344cafe4ab2c3a1c3510b38d6" +dependencies = [ + "arrow 57.0.0", + "async-trait", + "bytes", + "chrono", + "datafusion-common 51.0.0", + "datafusion-common-runtime 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-adapter 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-plan 51.0.0", + "datafusion-session 51.0.0", "futures", "glob", "itertools 0.14.0", "log", "object_store", - "parquet", "rand 0.9.2", - "tempfile", "tokio", "url", ] +[[package]] +name = "datafusion-datasource-arrow" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804fa9b4ecf3157982021770617200ef7c1b2979d57bec9044748314775a9aea" +dependencies = [ + "arrow 57.0.0", + "arrow-ipc 57.0.0", + "async-trait", + "bytes", + "datafusion-common 51.0.0", + "datafusion-common-runtime 51.0.0", + "datafusion-datasource 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-plan 51.0.0", + "datafusion-session 51.0.0", + "futures", + "itertools 0.14.0", + "object_store", + "tokio", +] + [[package]] name = "datafusion-datasource-csv" version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64533a90f78e1684bfb113d200b540f18f268134622d7c96bbebc91354d04825" dependencies = [ - "arrow", + "arrow 56.2.0", "async-trait", "bytes", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-catalog 50.3.0", + "datafusion-common 50.3.0", + "datafusion-common-runtime 50.3.0", + "datafusion-datasource 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "datafusion-physical-plan 50.3.0", + "datafusion-session 50.3.0", + "futures", + "object_store", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61a1641a40b259bab38131c5e6f48fac0717bedb7dc93690e604142a849e0568" +dependencies = [ + "arrow 57.0.0", + "async-trait", + "bytes", + "datafusion-common 51.0.0", + "datafusion-common-runtime 51.0.0", + "datafusion-datasource 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-plan 51.0.0", + "datafusion-session 51.0.0", "futures", "object_store", "regex", @@ -2261,55 +2689,74 @@ version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d7ebeb12c77df0aacad26f21b0d033aeede423a64b2b352f53048a75bf1d6e6" dependencies = [ - "arrow", + "arrow 56.2.0", "async-trait", "bytes", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-catalog 50.3.0", + "datafusion-common 50.3.0", + "datafusion-common-runtime 50.3.0", + "datafusion-datasource 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "datafusion-physical-plan 50.3.0", + "datafusion-session 50.3.0", "futures", "object_store", "serde_json", "tokio", ] +[[package]] +name = "datafusion-datasource-json" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adeacdb00c1d37271176f8fb6a1d8ce096baba16ea7a4b2671840c5c9c64fe85" +dependencies = [ + "arrow 57.0.0", + "async-trait", + "bytes", + "datafusion-common 51.0.0", + "datafusion-common-runtime 51.0.0", + "datafusion-datasource 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-plan 51.0.0", + "datafusion-session 51.0.0", + "futures", + "object_store", + "tokio", +] + [[package]] name = "datafusion-datasource-parquet" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09e783c4c7d7faa1199af2df4761c68530634521b176a8d1331ddbc5a5c75133" +checksum = "43d0b60ffd66f28bfb026565d62b0a6cbc416da09814766a3797bba7d85a3cd9" dependencies = [ - "arrow", + "arrow 57.0.0", "async-trait", "bytes", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-pruning", - "datafusion-session", + "datafusion-common 51.0.0", + "datafusion-common-runtime 51.0.0", + "datafusion-datasource 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-functions-aggregate-common 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-adapter 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-plan 51.0.0", + "datafusion-pruning 51.0.0", + "datafusion-session 51.0.0", "futures", "itertools 0.14.0", "log", "object_store", "parking_lot", "parquet", - "rand 0.9.2", "tokio", ] @@ -2319,17 +2766,43 @@ version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99ee6b1d9a80d13f9deb2291f45c07044b8e62fb540dbde2453a18be17a36429" +[[package]] +name = "datafusion-doc" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b99e13947667b36ad713549237362afb054b2d8f8cc447751e23ec61202db07" + [[package]] name = "datafusion-execution" version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4cec0a57653bec7b933fb248d3ffa3fa3ab3bd33bd140dc917f714ac036f531" dependencies = [ - "arrow", + "arrow 56.2.0", "async-trait", "dashmap", - "datafusion-common", - "datafusion-expr", + "datafusion-common 50.3.0", + "datafusion-expr 50.3.0", + "futures", + "log", + "object_store", + "parking_lot", + "rand 0.9.2", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-execution" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63695643190679037bc946ad46a263b62016931547bf119859c511f7ff2f5178" +dependencies = [ + "arrow 57.0.0", + "async-trait", + "dashmap", + "datafusion-common 51.0.0", + "datafusion-expr 51.0.0", "futures", "log", "object_store", @@ -2345,19 +2818,41 @@ version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef76910bdca909722586389156d0aa4da4020e1631994d50fadd8ad4b1aa05fe" dependencies = [ - "arrow", + "arrow 56.2.0", "async-trait", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", + "datafusion-common 50.3.0", + "datafusion-doc 50.3.0", + "datafusion-expr-common 50.3.0", + "datafusion-functions-aggregate-common 50.3.0", + "datafusion-functions-window-common 50.3.0", + "datafusion-physical-expr-common 50.3.0", "indexmap", "paste", "serde_json", - "sqlparser", + "sqlparser 0.58.0", +] + +[[package]] +name = "datafusion-expr" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a4787cbf5feb1ab351f789063398f67654a6df75c4d37d7f637dc96f951a91" +dependencies = [ + "arrow 57.0.0", + "async-trait", + "chrono", + "datafusion-common 51.0.0", + "datafusion-doc 51.0.0", + "datafusion-expr-common 51.0.0", + "datafusion-functions-aggregate-common 51.0.0", + "datafusion-functions-window-common 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "indexmap", + "itertools 0.14.0", + "paste", + "serde_json", + "sqlparser 0.59.0", ] [[package]] @@ -2366,8 +2861,21 @@ version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d155ccbda29591ca71a1344dd6bed26c65a4438072b400df9db59447f590bb6" dependencies = [ - "arrow", - "datafusion-common", + "arrow 56.2.0", + "datafusion-common 50.3.0", + "indexmap", + "itertools 0.14.0", + "paste", +] + +[[package]] +name = "datafusion-expr-common" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce2fb1b8c15c9ac45b0863c30b268c69dc9ee7a1ee13ecf5d067738338173dc" +dependencies = [ + "arrow 57.0.0", + "datafusion-common 51.0.0", "indexmap", "itertools 0.14.0", "paste", @@ -2379,18 +2887,18 @@ version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7de2782136bd6014670fd84fe3b0ca3b3e4106c96403c3ae05c0598577139977" dependencies = [ - "arrow", - "arrow-buffer", + "arrow 56.2.0", + "arrow-buffer 56.2.0", "base64", "blake2", "blake3", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-macros", + "datafusion-common 50.3.0", + "datafusion-doc 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-expr-common 50.3.0", + "datafusion-macros 50.3.0", "hex", "itertools 0.14.0", "log", @@ -2402,6 +2910,32 @@ dependencies = [ "uuid", ] +[[package]] +name = "datafusion-functions" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "794a9db7f7b96b3346fc007ff25e994f09b8f0511b4cf7dff651fadfe3ebb28f" +dependencies = [ + "arrow 57.0.0", + "arrow-buffer 57.0.0", + "base64", + "chrono", + "datafusion-common 51.0.0", + "datafusion-doc 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-expr-common 51.0.0", + "datafusion-macros 51.0.0", + "hex", + "itertools 0.14.0", + "log", + "num-traits", + "rand 0.9.2", + "regex", + "unicode-segmentation", + "uuid", +] + [[package]] name = "datafusion-functions-aggregate" version = "50.3.0" @@ -2409,15 +2943,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07331fc13603a9da97b74fd8a273f4238222943dffdbbed1c4c6f862a30105bf" dependencies = [ "ahash", - "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "arrow 56.2.0", + "datafusion-common 50.3.0", + "datafusion-doc 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-functions-aggregate-common 50.3.0", + "datafusion-macros 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "half", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c25210520a9dcf9c2b2cbbce31ebd4131ef5af7fc60ee92b266dc7d159cb305" +dependencies = [ + "ahash", + "arrow 57.0.0", + "datafusion-common 51.0.0", + "datafusion-doc 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-functions-aggregate-common 51.0.0", + "datafusion-macros 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-common 51.0.0", "half", "log", "paste", @@ -2430,29 +2985,65 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5951e572a8610b89968a09b5420515a121fbc305c0258651f318dc07c97ab17" dependencies = [ "ahash", - "arrow", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", + "arrow 56.2.0", + "datafusion-common 50.3.0", + "datafusion-expr-common 50.3.0", + "datafusion-physical-expr-common 50.3.0", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62f4a66f3b87300bb70f4124b55434d2ae3fe80455f3574701d0348da040b55d" +dependencies = [ + "ahash", + "arrow 57.0.0", + "datafusion-common 51.0.0", + "datafusion-expr-common 51.0.0", + "datafusion-physical-expr-common 51.0.0", +] + +[[package]] +name = "datafusion-functions-nested" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdacca9302c3d8fc03f3e94f338767e786a88a33f5ebad6ffc0e7b50364b9ea3" +dependencies = [ + "arrow 56.2.0", + "arrow-ord 56.2.0", + "datafusion-common 50.3.0", + "datafusion-doc 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-functions 50.3.0", + "datafusion-functions-aggregate 50.3.0", + "datafusion-functions-aggregate-common 50.3.0", + "datafusion-macros 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "itertools 0.14.0", + "log", + "paste", ] [[package]] name = "datafusion-functions-nested" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdacca9302c3d8fc03f3e94f338767e786a88a33f5ebad6ffc0e7b50364b9ea3" -dependencies = [ - "arrow", - "arrow-ord", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr-common", +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae5c06eed03918dc7fe7a9f082a284050f0e9ecf95d72f57712d1496da03b8c4" +dependencies = [ + "arrow 57.0.0", + "arrow-ord 57.0.0", + "datafusion-common 51.0.0", + "datafusion-doc 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-expr-common 51.0.0", + "datafusion-functions 51.0.0", + "datafusion-functions-aggregate 51.0.0", + "datafusion-functions-aggregate-common 51.0.0", + "datafusion-macros 51.0.0", + "datafusion-physical-expr-common 51.0.0", "itertools 0.14.0", "log", "paste", @@ -2464,12 +3055,28 @@ version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c37ff8a99434fbbad604a7e0669717c58c7c4f14c472d45067c4b016621d981" dependencies = [ - "arrow", + "arrow 56.2.0", + "async-trait", + "datafusion-catalog 50.3.0", + "datafusion-common 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-physical-plan 50.3.0", + "parking_lot", + "paste", +] + +[[package]] +name = "datafusion-functions-table" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db4fed1d71738fbe22e2712d71396db04c25de4111f1ec252b8f4c6d3b25d7f5" +dependencies = [ + "arrow 57.0.0", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-plan", + "datafusion-catalog 51.0.0", + "datafusion-common 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-physical-plan 51.0.0", "parking_lot", "paste", ] @@ -2480,14 +3087,32 @@ version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48e2aea7c79c926cffabb13dc27309d4eaeb130f4a21c8ba91cdd241c813652b" dependencies = [ - "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "arrow 56.2.0", + "datafusion-common 50.3.0", + "datafusion-doc 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-functions-window-common 50.3.0", + "datafusion-macros 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-window" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d92206aa5ae21892f1552b4d61758a862a70956e6fd7a95cb85db1de74bc6d1" +dependencies = [ + "arrow 57.0.0", + "datafusion-common 51.0.0", + "datafusion-doc 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-functions-window-common 51.0.0", + "datafusion-macros 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-common 51.0.0", "log", "paste", ] @@ -2498,8 +3123,18 @@ version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fead257ab5fd2ffc3b40fda64da307e20de0040fe43d49197241d9de82a487f" dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", + "datafusion-common 50.3.0", + "datafusion-physical-expr-common 50.3.0", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53ae9bcc39800820d53a22d758b3b8726ff84a5a3e24cecef04ef4e5fdf1c7cc" +dependencies = [ + "datafusion-common 51.0.0", + "datafusion-physical-expr-common 51.0.0", ] [[package]] @@ -2508,9 +3143,20 @@ version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec6f637bce95efac05cdfb9b6c19579ed4aa5f6b94d951cfa5bb054b7bb4f730" dependencies = [ - "datafusion-expr", + "datafusion-expr 50.3.0", + "quote", + "syn 2.0.110", +] + +[[package]] +name = "datafusion-macros" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1063ad4c9e094b3f798acee16d9a47bd7372d9699be2de21b05c3bd3f34ab848" +dependencies = [ + "datafusion-doc 51.0.0", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -2519,12 +3165,31 @@ version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6583ef666ae000a613a837e69e456681a9faa96347bf3877661e9e89e141d8a" dependencies = [ - "arrow", + "arrow 56.2.0", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", + "datafusion-common 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-expr-common 50.3.0", + "datafusion-physical-expr 50.3.0", + "indexmap", + "itertools 0.14.0", + "log", + "regex", + "regex-syntax", +] + +[[package]] +name = "datafusion-optimizer" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f35f9ec5d08b87fd1893a30c2929f2559c2f9806ca072d8fefca5009dc0f06a" +dependencies = [ + "arrow 57.0.0", + "chrono", + "datafusion-common 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-expr-common 51.0.0", + "datafusion-physical-expr 51.0.0", "indexmap", "itertools 0.14.0", "log", @@ -2539,12 +3204,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8668103361a272cbbe3a61f72eca60c9b7c706e87cc3565bcf21e2b277b84f6" dependencies = [ "ahash", - "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", + "arrow 56.2.0", + "datafusion-common 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-expr-common 50.3.0", + "datafusion-functions-aggregate-common 50.3.0", + "datafusion-physical-expr-common 50.3.0", "half", "hashbrown 0.14.5", "indexmap", @@ -2555,18 +3220,55 @@ dependencies = [ "petgraph 0.8.3", ] +[[package]] +name = "datafusion-physical-expr" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c30cc8012e9eedcb48bbe112c6eff4ae5ed19cf3003cb0f505662e88b7014c5d" +dependencies = [ + "ahash", + "arrow 57.0.0", + "datafusion-common 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-expr-common 51.0.0", + "datafusion-functions-aggregate-common 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "half", + "hashbrown 0.14.5", + "indexmap", + "itertools 0.14.0", + "parking_lot", + "paste", + "petgraph 0.8.3", +] + [[package]] name = "datafusion-physical-expr-adapter" version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "815acced725d30601b397e39958e0e55630e0a10d66ef7769c14ae6597298bb0" dependencies = [ - "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "arrow 56.2.0", + "datafusion-common 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-functions 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-expr-adapter" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f9ff2dbd476221b1f67337699eff432781c4e6e1713d2aefdaa517dfbf79768" +dependencies = [ + "arrow 57.0.0", + "datafusion-common 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-functions 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-common 51.0.0", "itertools 0.14.0", ] @@ -2577,9 +3279,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6652fe7b5bf87e85ed175f571745305565da2c0b599d98e697bcbedc7baa47c3" dependencies = [ "ahash", - "arrow", - "datafusion-common", - "datafusion-expr-common", + "arrow 56.2.0", + "datafusion-common 50.3.0", + "datafusion-expr-common 50.3.0", + "hashbrown 0.14.5", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90da43e1ec550b172f34c87ec68161986ced70fd05c8d2a2add66eef9c276f03" +dependencies = [ + "ahash", + "arrow 57.0.0", + "datafusion-common 51.0.0", + "datafusion-expr-common 51.0.0", "hashbrown 0.14.5", "itertools 0.14.0", ] @@ -2590,19 +3306,37 @@ version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49b7d623eb6162a3332b564a0907ba00895c505d101b99af78345f1acf929b5c" dependencies = [ - "arrow", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", + "arrow 56.2.0", + "datafusion-common 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-expr-common 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "datafusion-physical-plan 50.3.0", + "datafusion-pruning 50.3.0", "itertools 0.14.0", "log", ] +[[package]] +name = "datafusion-physical-optimizer" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce9804f799acd7daef3be7aaffe77c0033768ed8fdbf5fb82fc4c5f2e6bc14e6" +dependencies = [ + "arrow 57.0.0", + "datafusion-common 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-expr-common 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-plan 51.0.0", + "datafusion-pruning 51.0.0", + "itertools 0.14.0", +] + [[package]] name = "datafusion-physical-plan" version = "50.3.0" @@ -2610,19 +3344,50 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2f7f778a1a838dec124efb96eae6144237d546945587557c9e6936b3414558c" dependencies = [ "ahash", - "arrow", - "arrow-ord", - "arrow-schema", + "arrow 56.2.0", + "arrow-ord 56.2.0", + "arrow-schema 56.2.0", + "async-trait", + "chrono", + "datafusion-common 50.3.0", + "datafusion-common-runtime 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-functions-aggregate-common 50.3.0", + "datafusion-functions-window-common 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "futures", + "half", + "hashbrown 0.14.5", + "indexmap", + "itertools 0.14.0", + "log", + "parking_lot", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "datafusion-physical-plan" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0acf0ad6b6924c6b1aa7d213b181e012e2d3ec0a64ff5b10ee6282ab0f8532ac" +dependencies = [ + "ahash", + "arrow 57.0.0", + "arrow-ord 57.0.0", + "arrow-schema 57.0.0", "async-trait", "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 51.0.0", + "datafusion-common-runtime 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-functions-aggregate-common 51.0.0", + "datafusion-functions-window-common 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-common 51.0.0", "futures", "half", "hashbrown 0.14.5", @@ -2640,14 +3405,31 @@ version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd1e59e2ca14fe3c30f141600b10ad8815e2856caa59ebbd0e3e07cd3d127a65" dependencies = [ - "arrow", - "arrow-schema", - "datafusion-common", - "datafusion-datasource", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", + "arrow 56.2.0", + "arrow-schema 56.2.0", + "datafusion-common 50.3.0", + "datafusion-datasource 50.3.0", + "datafusion-expr-common 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "datafusion-physical-plan 50.3.0", + "itertools 0.14.0", + "log", +] + +[[package]] +name = "datafusion-pruning" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2c2498a1f134a9e11a9f5ed202a2a7d7e9774bd9249295593053ea3be999db" +dependencies = [ + "arrow 57.0.0", + "datafusion-common 51.0.0", + "datafusion-datasource 51.0.0", + "datafusion-expr-common 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-plan 51.0.0", "itertools 0.14.0", "log", ] @@ -2658,16 +3440,16 @@ version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21ef8e2745583619bd7a49474e8f45fbe98ebb31a133f27802217125a7b3d58d" dependencies = [ - "arrow", + "arrow 56.2.0", "async-trait", "dashmap", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-sql", + "datafusion-common 50.3.0", + "datafusion-common-runtime 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-plan 50.3.0", + "datafusion-sql 50.3.0", "futures", "itertools 0.14.0", "log", @@ -2676,20 +3458,51 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-session" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f96eebd17555386f459037c65ab73aae8df09f464524c709d6a3134ad4f4776" +dependencies = [ + "async-trait", + "datafusion-common 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-physical-plan 51.0.0", + "parking_lot", +] + [[package]] name = "datafusion-sql" version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89abd9868770386fede29e5a4b14f49c0bf48d652c3b9d7a8a0332329b87d50b" dependencies = [ - "arrow", + "arrow 56.2.0", + "bigdecimal", + "datafusion-common 50.3.0", + "datafusion-expr 50.3.0", + "indexmap", + "log", + "regex", + "sqlparser 0.58.0", +] + +[[package]] +name = "datafusion-sql" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fc195fe60634b2c6ccfd131b487de46dc30eccae8a3c35a13f136e7f440414f" +dependencies = [ + "arrow 57.0.0", "bigdecimal", - "datafusion-common", - "datafusion-expr", + "chrono", + "datafusion-common 51.0.0", + "datafusion-expr 51.0.0", "indexmap", "log", "regex", - "sqlparser", + "sqlparser 0.59.0", ] [[package]] @@ -2731,9 +3544,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a41953f86f8a05768a6cda24def994fd2f424b04ec5c719cf89989779f199071" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" dependencies = [ "powerfmt", "serde_core", @@ -2747,7 +3560,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -2768,7 +3581,7 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -2812,7 +3625,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -2823,7 +3636,7 @@ checksum = "8dc51d98e636f5e3b0759a39257458b22619cac7e96d932da6eeb052891bb67c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -2837,9 +3650,9 @@ dependencies = [ [[package]] name = "document-features" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" dependencies = [ "litrs", ] @@ -2900,7 +3713,7 @@ checksum = "685adfa4d6f3d765a26bc5dbc936577de9abf756c1feeb3089b01dd395034842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -2920,14 +3733,14 @@ checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] name = "env_filter" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +checksum = "1bf3c259d255ca70051b30e2e95b5446cdb8949ac4cd22c0d7fd634d89f568e2" dependencies = [ "log", "regex", @@ -3002,9 +3815,9 @@ dependencies = [ [[package]] name = "exponential-decay-histogram" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "988360e80225a42d5e2f78fcb90ef7d54d1e4394d335476b5465d34942426776" +checksum = "d7962d7e9baab6ea05af175491fa6a8441f3bf461558037622142573d98dd6d6" dependencies = [ "ordered-float 5.1.0", "rand 0.9.2", @@ -3119,9 +3932,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc5a4e564e38c699f2880d3fda590bedc2e69f3f84cd48b457bd892ce61d0aa9" +checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" dependencies = [ "crc32fast", "libz-rs-sys", @@ -3177,7 +3990,7 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d2475ce218217196b161b025598f77e2b405d5e729f7c37bfff145f5df00a41" dependencies = [ - "arrow-array", + "arrow-array 56.2.0", "rand 0.9.2", ] @@ -3271,7 +4084,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -3335,9 +4148,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.8" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dc8f7d2ded5f9209535e4b3fd4d39c002f30902ff5ce9f64e2c33d549576500" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -3506,11 +4319,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.11" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3598,9 +4411,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hyper" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ "atomic-waker", "bytes", @@ -3638,9 +4451,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" +checksum = "52e9a2a24dc5c6821e71a7030e1e14b7b632acac55c40e9d2e082c621261bb56" dependencies = [ "base64", "bytes", @@ -3683,7 +4496,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core", + "windows-core 0.62.2", ] [[package]] @@ -3697,9 +4510,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" dependencies = [ "displaydoc", "potential_utf", @@ -3710,9 +4523,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" dependencies = [ "displaydoc", "litemap", @@ -3723,11 +4536,10 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" dependencies = [ - "displaydoc", "icu_collections", "icu_normalizer_data", "icu_properties", @@ -3738,42 +4550,38 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" [[package]] name = "icu_properties" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99" dependencies = [ - "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", "icu_provider", - "potential_utf", "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" +checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899" [[package]] name = "icu_provider" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" dependencies = [ "displaydoc", "icu_locale_core", - "stable_deref_trait", - "tinystr", "writeable", "yoke", "zerofrom", @@ -3825,17 +4633,17 @@ checksum = "a0eb5a3343abf848c0984fe4604b2b105da9539376e24fc0a3b0007411ae4fd9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] name = "indexmap" -version = "2.11.4" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" +checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" dependencies = [ "equivalent", - "hashbrown 0.15.5", + "hashbrown 0.16.1", ] [[package]] @@ -3854,9 +4662,12 @@ dependencies = [ [[package]] name = "indoc" -version = "2.0.6" +version = "2.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] [[package]] name = "inout" @@ -3870,9 +4681,9 @@ dependencies = [ [[package]] name = "insta" -version = "1.44.3" +version = "1.44.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5c943d4415edd8153251b6f197de5eb1640e56d84e8d9159bea190421c73698" +checksum = "e8732d3774162a0851e3f2b150eb98f31a9885dd75985099421d393385a01dfd" dependencies = [ "console 0.15.11", "once_cell", @@ -3889,7 +4700,7 @@ dependencies = [ "indoc", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -3915,9 +4726,9 @@ checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "iri-string" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" dependencies = [ "memchr", "serde", @@ -3925,9 +4736,9 @@ dependencies = [ [[package]] name = "is_terminal_polyfill" -version = "1.70.1" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "itertools" @@ -3994,7 +4805,7 @@ checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -4058,9 +4869,9 @@ dependencies = [ [[package]] name = "jsonb" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a452366d21e8d3cbca680c41388e01d6a88739afef7877961946a6da409f9ccd" +checksum = "2a901f06163d352fbe41c3c2ff5e08b75330a003cc941e988fb501022f5421e6" dependencies = [ "byteorder", "ethnum", @@ -4107,15 +4918,15 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2f0ca022d0424d991933a62d2898864cf5621873962bd84e65e7d1f023f9c36" dependencies = [ - "arrow", - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-ipc", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", + "arrow 56.2.0", + "arrow-arith 56.2.0", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-ipc 56.2.0", + "arrow-ord 56.2.0", + "arrow-row 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "async-recursion", "async-trait", "async_cell", @@ -4124,11 +4935,11 @@ dependencies = [ "bytes", "chrono", "dashmap", - "datafusion", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-plan", + "datafusion 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-functions 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-plan 50.3.0", "deepsize", "either", "futures", @@ -4172,12 +4983,12 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7552f8d528775bf0ab21e1f75dcb70bdb2a828eeae58024a803b5a4655fd9a11" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "bytes", "getrandom 0.2.16", "half", @@ -4203,15 +5014,15 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69c752dedd207384892006c40930f898d6634e05e3d489e89763abfe4b9307e7" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-schema 56.2.0", "async-trait", "byteorder", "bytes", "chrono", - "datafusion-common", - "datafusion-sql", + "datafusion-common 50.3.0", + "datafusion-sql 50.3.0", "deepsize", "futures", "lance-arrow", @@ -4241,18 +5052,18 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21e1e98ca6e5cd337bdda2d9fb66063f295c0c2852d2bc6831366fea833ee608" dependencies = [ - "arrow", - "arrow-array", - "arrow-buffer", - "arrow-ord", - "arrow-schema", - "arrow-select", + "arrow 56.2.0", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-ord 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "async-trait", "chrono", - "datafusion", - "datafusion-common", - "datafusion-functions", - "datafusion-physical-expr", + "datafusion 50.3.0", + "datafusion-common 50.3.0", + "datafusion-functions 50.3.0", + "datafusion-physical-expr 50.3.0", "futures", "jsonb", "lance-arrow", @@ -4272,10 +5083,10 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "483c643fc2806ed1a2766edf4d180511bbd1d549bcc60373e33f4785c6185891" dependencies = [ - "arrow", - "arrow-array", - "arrow-cast", - "arrow-schema", + "arrow 56.2.0", + "arrow-array 56.2.0", + "arrow-cast 56.2.0", + "arrow-schema 56.2.0", "chrono", "futures", "half", @@ -4291,13 +5102,13 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a199d1fa3487529c5ffc433fbd1721231330b9350c2ff9b0c7b7dbdb98f0806a" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-arith 56.2.0", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "bytemuck", "byteorder", "bytes", @@ -4330,17 +5141,17 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b57def2279465232cf5a8cd996300c632442e368745768bbed661c7f0a35334b" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-arith 56.2.0", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "async-recursion", "async-trait", "byteorder", "bytes", - "datafusion-common", + "datafusion-common 50.3.0", "deepsize", "futures", "lance-arrow", @@ -4364,12 +5175,12 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75938c61e986aef8c615dc44c92e4c19e393160a59e2b57402ccfe08c5e63af" dependencies = [ - "arrow", - "arrow-arith", - "arrow-array", - "arrow-ord", - "arrow-schema", - "arrow-select", + "arrow 56.2.0", + "arrow-arith 56.2.0", + "arrow-array 56.2.0", + "arrow-ord 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "async-channel", "async-recursion", "async-trait", @@ -4377,11 +5188,11 @@ dependencies = [ "bitvec", "bytes", "crossbeam-queue", - "datafusion", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-sql", + "datafusion 50.3.0", + "datafusion-common 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-sql 50.3.0", "deepsize", "dirs", "fst", @@ -4427,14 +5238,14 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa6c3b5b28570d6c951206c5b043f1b35c936928af14fca6f2ac25b0097e4c32" dependencies = [ - "arrow", - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow 56.2.0", + "arrow-arith 56.2.0", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "async-recursion", "async-trait", "aws-config", @@ -4469,9 +5280,9 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3cbc7e85a89ff9cb3a4627559dea3fd1c1fb16c0d8bc46ede75eefef51eec06" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-schema 56.2.0", "cc", "deepsize", "half", @@ -4487,7 +5298,7 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897dd6726816515bb70a698ce7cda44670dca5761637696d7905b45f405a8cd9" dependencies = [ - "arrow", + "arrow 56.2.0", "async-trait", "bytes", "lance-core", @@ -4514,11 +5325,11 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8facc13760ba034b6c38767b16adba85e44cbcbea8124dc0c63c43865c60630" dependencies = [ - "arrow", - "arrow-array", - "arrow-buffer", - "arrow-ipc", - "arrow-schema", + "arrow 56.2.0", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-ipc 56.2.0", + "arrow-schema 56.2.0", "async-trait", "byteorder", "bytes", @@ -4745,15 +5556,15 @@ checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "litemap" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" [[package]] name = "litrs" -version = "0.4.2" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5e54036fe321fd421e10d732f155734c4e4afd610dd556d9a82833ab3ee0bed" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" [[package]] name = "lock_api" @@ -4960,14 +5771,14 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" dependencies = [ "libc", "log", "wasi", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5021,7 +5832,7 @@ checksum = "b093064383341eb3271f42e381cb8f10a01459478446953953c75d24bd339fc0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", "target-features", ] @@ -5217,11 +6028,10 @@ dependencies = [ [[package]] name = "num-bigint-dig" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" dependencies = [ - "byteorder", "lazy_static", "libm", "num-integer", @@ -5316,7 +6126,7 @@ checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -5368,7 +6178,7 @@ dependencies = [ "md-5", "parking_lot", "percent-encoding", - "quick-xml 0.38.3", + "quick-xml 0.38.4", "rand 0.9.2", "reqwest", "ring", @@ -5387,9 +6197,9 @@ dependencies = [ [[package]] name = "object_store_opendal" -version = "0.54.0" +version = "0.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce697ee723fdc3eaf6c457abf4059034be15167022b18b619993802cd1443d5" +checksum = "c0b88fc0e0c4890c1d99e2b8c519c5db40f7d9b69a0f562ff1ad4967a4c8bbc6" dependencies = [ "async-trait", "bytes", @@ -5411,9 +6221,9 @@ dependencies = [ [[package]] name = "once_cell_polyfill" -version = "1.70.1" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "oneshot" @@ -5423,9 +6233,9 @@ checksum = "b4ce411919553d3f9fa53a0880544cda985a112117a0444d5ff1e870a893d6ea" [[package]] name = "opendal" -version = "0.54.0" +version = "0.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb9838d0575c6dbaf3fcec7255af8d5771996d4af900bbb6fa9a314dec00a1a" +checksum = "42afda58fa2cf50914402d132cc1caacff116a85d10c72ab2082bb7c50021754" dependencies = [ "anyhow", "backon", @@ -5440,7 +6250,7 @@ dependencies = [ "log", "md-5", "percent-encoding", - "quick-xml 0.37.5", + "quick-xml 0.38.4", "reqsign", "reqwest", "serde", @@ -5613,7 +6423,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -5647,18 +6457,18 @@ dependencies = [ [[package]] name = "parquet" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0dbd48ad52d7dccf8ea1b90a3ddbfaea4f69878dd7683e51c507d4bc52b5b27" +checksum = "7a0f31027ef1af7549f7cec603a9a21dce706d3f8d7c2060a68f43c1773be95a" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ipc", - "arrow-schema", - "arrow-select", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-cast 57.0.0", + "arrow-data 57.0.0", + "arrow-ipc 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", "base64", "brotli", "bytes", @@ -5668,11 +6478,11 @@ dependencies = [ "half", "hashbrown 0.16.1", "lz4_flex", - "num", "num-bigint", + "num-integer", + "num-traits", "object_store", "paste", - "ring", "seq-macro", "simdutf8", "snap", @@ -5810,7 +6620,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -5917,9 +6727,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" dependencies = [ "zerovec", ] @@ -5932,9 +6742,9 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppmd-rust" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c834641d8ad1b348c9ee86dec3b9840d805acd5f24daa5f90c788951a52ff59b" +checksum = "d558c559f0450f16f2a27a1f017ef38468c1090c9ce63c8e51366232d53717b4" [[package]] name = "ppv-lite86" @@ -5952,7 +6762,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -5977,9 +6787,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.101" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" dependencies = [ "unicode-ident", ] @@ -6030,7 +6840,7 @@ dependencies = [ "prost 0.13.5", "prost-types 0.13.5", "regex", - "syn 2.0.106", + "syn 2.0.110", "tempfile", ] @@ -6050,7 +6860,7 @@ dependencies = [ "prost 0.14.1", "prost-types 0.14.1", "regex", - "syn 2.0.106", + "syn 2.0.110", "tempfile", ] @@ -6064,7 +6874,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -6077,7 +6887,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -6090,7 +6900,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -6177,7 +6987,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -6190,7 +7000,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -6205,9 +7015,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.38.3" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42a232e7487fc2ef313d96dde7948e7a3c05101870d8985e4fd8d26aedd27b89" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" dependencies = [ "memchr", "serde", @@ -6270,9 +7080,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.41" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" dependencies = [ "proc-macro2", ] @@ -6401,9 +7211,9 @@ dependencies = [ [[package]] name = "rangemap" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93e7e49bb0bf967717f7bd674458b3d6b0c5f48ec7e3038166026a69fc22223" +checksum = "acbbbbea733ec66275512d0b9694f34102e7d5406fdbe2ad8d21b28dce92887c" [[package]] name = "ratatui" @@ -6486,9 +7296,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "722166aa0d7438abbaa4d5cc2c649dac844e8c56d82fb3d33e9c34b5cd268fc6" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", @@ -6503,9 +7313,9 @@ checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" [[package]] name = "regex-syntax" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3160422bbd54dd5ecfdca71e5fd59b7b8fe2b1697ab2baf64f6d05dcc66d298" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "relative-path" @@ -6629,9 +7439,9 @@ dependencies = [ [[package]] name = "rsa" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78928ac1ed176a5ca1d17e578a1825f3d81ca54cf41053a592584b020cfd691b" +checksum = "40a0376c50d0358279d9d643e4bf7b7be212f1f4ff1da9070a7b54d22ef75c88" dependencies = [ "const-oid", "digest", @@ -6673,7 +7483,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.106", + "syn 2.0.110", "unicode-ident", ] @@ -6685,7 +7495,7 @@ checksum = "b3a8fb4672e840a587a66fc577a5491375df51ddb88f2a2c2a792598c326fe14" dependencies = [ "quote", "rand 0.8.5", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -6757,9 +7567,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.32" +version = "0.23.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd3c25631629d034ce7cd9940adc9d45762d46de2b0f57193c4443b92c6d4d40" +checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" dependencies = [ "aws-lc-rs", "once_cell", @@ -6772,9 +7582,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -6793,9 +7603,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +checksum = "94182ad936a0c91c324cd46c6511b9510ed16af436d7b5bab34beab0afd55f7a" dependencies = [ "web-time", "zeroize", @@ -6803,9 +7613,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.7" +version = "0.103.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10b3f4191e8a80e6b43eebabfac91e5dcecebb27a71f04e820c47ec41d314bf" +checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" dependencies = [ "aws-lc-rs", "ring", @@ -6953,7 +7763,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -6977,7 +7787,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -7068,9 +7878,9 @@ dependencies = [ [[package]] name = "signal-hook-mio" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" dependencies = [ "libc", "mio", @@ -7079,9 +7889,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.6" +version = "1.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" +checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" dependencies = [ "libc", ] @@ -7222,7 +8032,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -7233,12 +8043,12 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -7267,6 +8077,16 @@ dependencies = [ "sqlparser_derive", ] +[[package]] +name = "sqlparser" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4591acadbcf52f0af60eafbb2c003232b2b4cd8de5f0e9437cb8b1b59046cc0f" +dependencies = [ + "log", + "sqlparser_derive", +] + [[package]] name = "sqlparser_derive" version = "0.3.0" @@ -7275,7 +8095,7 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -7346,7 +8166,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -7358,7 +8178,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -7380,9 +8200,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.106" +version = "2.0.110" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea" dependencies = [ "proc-macro2", "quote", @@ -7406,7 +8226,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -7726,7 +8546,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -7737,7 +8557,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -7823,9 +8643,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" dependencies = [ "displaydoc", "zerovec", @@ -7871,7 +8691,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -8048,7 +8868,7 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9e70da6e86f54ff3524628dc84cb0a9e674ea28e8a8a82fe3839af8c7fd743b" dependencies = [ - "arrow", + "arrow 56.2.0", "tpchgen", ] @@ -8071,7 +8891,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -8177,9 +8997,9 @@ checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-ident" -version = "1.0.19" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "unicode-segmentation" @@ -8224,9 +9044,9 @@ checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" [[package]] name = "unit-prefix" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "323402cff2dd658f39ca17c789b502021b3f18707c91cdf22e3838e1b4023817" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" [[package]] name = "untrusted" @@ -8299,7 +9119,7 @@ name = "vortex" version = "0.1.0" dependencies = [ "anyhow", - "arrow-array", + "arrow-array 57.0.0", "codspeed-divan-compat", "fastlanes", "itertools 0.14.0", @@ -8371,15 +9191,15 @@ version = "0.1.0" dependencies = [ "arbitrary", "arcref", - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ord", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-arith 57.0.0", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-cast 57.0.0", + "arrow-data 57.0.0", + "arrow-ord 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", + "arrow-string 57.0.0", "async-trait", "bitvec", "cfg-if", @@ -8462,7 +9282,7 @@ dependencies = [ name = "vortex-buffer" version = "0.1.0" dependencies = [ - "arrow-buffer", + "arrow-buffer 57.0.0", "bitvec", "bytes", "codspeed-divan-compat", @@ -8496,9 +9316,9 @@ dependencies = [ name = "vortex-compute" version = "0.1.0" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-schema", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-schema 57.0.0", "codspeed-divan-compat", "log", "multiversion", @@ -8516,8 +9336,8 @@ name = "vortex-cxx" version = "0.1.0" dependencies = [ "anyhow", - "arrow-array", - "arrow-schema", + "arrow-array 57.0.0", + "arrow-schema 57.0.0", "async-fs", "cxx", "cxx-build", @@ -8532,22 +9352,22 @@ name = "vortex-datafusion" version = "0.1.0" dependencies = [ "anyhow", - "arrow-schema", + "arrow-schema 57.0.0", "async-trait", "chrono", - "datafusion", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", + "datafusion 51.0.0", + "datafusion-catalog 51.0.0", + "datafusion-common 51.0.0", + "datafusion-common-runtime 51.0.0", + "datafusion-datasource 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-functions 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-adapter 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-plan 51.0.0", + "datafusion-pruning 51.0.0", "futures", "insta", "itertools 0.14.0", @@ -8600,8 +9420,8 @@ name = "vortex-dtype" version = "0.1.0" dependencies = [ "arbitrary", - "arrow-buffer", - "arrow-schema", + "arrow-buffer 57.0.0", + "arrow-schema 57.0.0", "flatbuffers", "half", "insta", @@ -8662,7 +9482,7 @@ dependencies = [ name = "vortex-error" version = "0.1.0" dependencies = [ - "arrow-schema", + "arrow-schema 57.0.0", "flatbuffers", "jiff", "object_store", @@ -8678,7 +9498,7 @@ name = "vortex-fastlanes" version = "0.1.0" dependencies = [ "arrayref", - "arrow-buffer", + "arrow-buffer 57.0.0", "codspeed-divan-compat", "fastlanes", "itertools 0.14.0", @@ -8873,9 +9693,9 @@ dependencies = [ name = "vortex-jni" version = "0.1.0" dependencies = [ - "arrow-array", - "arrow-ipc", - "arrow-schema", + "arrow-array 57.0.0", + "arrow-ipc 57.0.0", + "arrow-schema 57.0.0", "futures", "jni", "log", @@ -8894,7 +9714,7 @@ name = "vortex-layout" version = "0.1.0" dependencies = [ "arcref", - "arrow-buffer", + "arrow-buffer 57.0.0", "async-stream", "async-trait", "flatbuffers", @@ -8987,9 +9807,9 @@ dependencies = [ name = "vortex-python" version = "0.1.0" dependencies = [ - "arrow-array", - "arrow-data", - "arrow-schema", + "arrow-array 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", "bytes", "itertools 0.14.0", "log", @@ -9008,8 +9828,8 @@ dependencies = [ name = "vortex-runend" version = "0.1.0" dependencies = [ - "arrow-array", - "arrow-buffer", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", "codspeed-divan-compat", "itertools 0.14.0", "num-traits", @@ -9029,7 +9849,7 @@ name = "vortex-scalar" version = "0.1.0" dependencies = [ "arbitrary", - "arrow-array", + "arrow-array 57.0.0", "bytes", "itertools 0.14.0", "num-traits", @@ -9049,8 +9869,8 @@ dependencies = [ name = "vortex-scan" version = "0.1.0" dependencies = [ - "arrow-array", - "arrow-schema", + "arrow-array 57.0.0", + "arrow-schema 57.0.0", "bit-vec", "futures", "itertools 0.14.0", @@ -9279,7 +10099,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", "wasm-bindgen-shared", ] @@ -9327,9 +10147,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b130c0d2d49f8b6889abc456e795e82525204f27c42cf767cf0d7734e089b8" +checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" dependencies = [ "rustls-pki-types", ] @@ -9372,7 +10192,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ "windows-collections", - "windows-core", + "windows-core 0.61.2", "windows-future", "windows-link 0.1.3", "windows-numerics", @@ -9384,7 +10204,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" dependencies = [ - "windows-core", + "windows-core 0.61.2", ] [[package]] @@ -9396,8 +10216,21 @@ dependencies = [ "windows-implement", "windows-interface", "windows-link 0.1.3", - "windows-result", - "windows-strings", + "windows-result 0.3.4", + "windows-strings 0.4.2", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", ] [[package]] @@ -9406,7 +10239,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ - "windows-core", + "windows-core 0.61.2", "windows-link 0.1.3", "windows-threading", ] @@ -9419,7 +10252,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -9430,7 +10263,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -9451,19 +10284,19 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" dependencies = [ - "windows-core", + "windows-core 0.61.2", "windows-link 0.1.3", ] [[package]] name = "windows-registry" -version = "0.5.3" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a9ed28765efc97bbc954883f4e6796c33a06546ebafacbabee9696967499e" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" dependencies = [ - "windows-link 0.1.3", - "windows-result", - "windows-strings", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", ] [[package]] @@ -9475,6 +10308,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-strings" version = "0.4.2" @@ -9484,6 +10326,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -9754,9 +10605,9 @@ dependencies = [ [[package]] name = "writeable" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" [[package]] name = "wyz" @@ -9822,11 +10673,10 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yoke" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" dependencies = [ - "serde", "stable_deref_trait", "yoke-derive", "zerofrom", @@ -9834,34 +10684,34 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.27" +version = "0.8.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +checksum = "43fa6694ed34d6e57407afbccdeecfa268c470a7d2a5b0cf49ce9fcc345afb90" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.27" +version = "0.8.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +checksum = "c640b22cd9817fae95be82f0d2f90b11f7605f6c319d16705c459b27ac2cbc26" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -9881,7 +10731,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", "synstructure", ] @@ -9902,14 +10752,14 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] name = "zerotrie" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" dependencies = [ "displaydoc", "yoke", @@ -9918,9 +10768,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.4" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" dependencies = [ "yoke", "zerofrom", @@ -9929,13 +10779,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -9982,9 +10832,9 @@ checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2" [[package]] name = "zopfli" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edfc5ee405f504cd4984ecc6f14d02d55cfda60fa4b689434ef4102aae150cd7" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" dependencies = [ "bumpalo", "crc32fast", diff --git a/Cargo.toml b/Cargo.toml index 7a676a4006e..16c7510e4d0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,16 +61,16 @@ anyhow = "1.0.95" arbitrary = "1.3.2" arcref = "0.2.0" arrayref = "0.3.7" -arrow-arith = "56" -arrow-array = "56" -arrow-buffer = "56" -arrow-cast = "56" -arrow-data = "56" -arrow-ipc = "56" -arrow-ord = "56" -arrow-schema = "56" -arrow-select = "56" -arrow-string = "56" +arrow-arith = "57" +arrow-array = "57" +arrow-buffer = "57" +arrow-cast = "57" +arrow-data = "57" +arrow-ipc = "57" +arrow-ord = "57" +arrow-schema = "57" +arrow-select = "57" +arrow-string = "57" async-compat = "0.2.5" async-fs = "2.2.0" async-stream = "0.3.6" @@ -83,7 +83,7 @@ bzip2 = "0.6.0" cbindgen = "0.29.0" cc = "1.2" cfg-if = "1" -chrono = "0.4.41" +chrono = "0.4.42" clap = "4.5" crossbeam-deque = "0.8.6" crossbeam-queue = "0.3.12" @@ -97,19 +97,19 @@ cudarc = { version = "0.18.0", features = [ "nvrtc", ], default-features = false } dashmap = "6.1.0" -datafusion = { version = "50", default-features = false } -datafusion-catalog = { version = "50" } -datafusion-common = { version = "50" } -datafusion-common-runtime = { version = "50" } -datafusion-datasource = { version = "50", default-features = false } -datafusion-execution = { version = "50" } -datafusion-expr = { version = "50" } -datafusion-functions = { version = "50" } -datafusion-physical-expr = { version = "50" } -datafusion-physical-expr-adapter = { version = "50" } -datafusion-physical-expr-common = { version = "50" } -datafusion-physical-plan = { version = "50" } -datafusion-pruning = { version = "50" } +datafusion = { version = "51", default-features = false, features = ["sql"] } +datafusion-catalog = { version = "51" } +datafusion-common = { version = "51" } +datafusion-common-runtime = { version = "51" } +datafusion-datasource = { version = "51", default-features = false } +datafusion-execution = { version = "51" } +datafusion-expr = { version = "51" } +datafusion-functions = { version = "51" } +datafusion-physical-expr = { version = "51" } +datafusion-physical-expr-adapter = { version = "51" } +datafusion-physical-expr-common = { version = "51" } +datafusion-physical-plan = { version = "51" } +datafusion-pruning = { version = "51" } dirs = "6.0.0" divan = { package = "codspeed-divan-compat", version = "4.0.4" } dyn-hash = "1.0.0" @@ -123,7 +123,7 @@ futures = { version = "0.3.31", default-features = false } fuzzy-matcher = "0.3" glob = "0.3.2" goldenfile = "1" -half = { version = "2.6", features = ["std", "num-traits"] } +half = { version = "2.7.1", features = ["std", "num-traits"] } hashbrown = "0.16.0" humansize = "2.1.3" indicatif = "0.18.0" @@ -144,14 +144,14 @@ noodles-bgzf = "0.44.0" noodles-vcf = "0.82.0" num-traits = "0.2.19" num_enum = { version = "0.7.3", default-features = false } -object_store = { version = "0.12.3", default-features = false } +object_store = { version = "0.12.4", default-features = false } once_cell = "1.21" oneshot = "0.1.11" opentelemetry = "0.31.0" opentelemetry-otlp = "0.31.0" opentelemetry_sdk = "0.31.0" parking_lot = { version = "0.12.3", features = ["nightly"] } -parquet = "56" +parquet = "57" paste = "1.0.15" pco = "0.4.4" pin-project-lite = "0.2.15" @@ -198,11 +198,11 @@ target-lexicon = "0.13" tempfile = "3" termtree = { version = "0.5" } thiserror = "2.0.3" -tokio = { version = "1.47" } +tokio = { version = "1.48" } tokio-stream = "0.1.17" tokio-util = { version = "0.7.16" } -tpchgen = { version = "2" } -tpchgen-arrow = { version = "2" } +tpchgen = { version = "2.0.1" } +tpchgen-arrow = { version = "2.0.1" } tracing = { version = "0.1.41" } tracing-perfetto = "0.1.5" tracing-subscriber = "0.3.20" diff --git a/bench-vortex/src/public_bi.rs b/bench-vortex/src/public_bi.rs index 4729787713d..a7253982ff5 100644 --- a/bench-vortex/src/public_bi.rs +++ b/bench-vortex/src/public_bi.rs @@ -432,7 +432,7 @@ impl PBIData { ListingOptions::new(df_format) .with_session_config_options(session.state().config()), ) - .with_schema(schema.into()); + .with_schema(schema.inner().clone()); let listing_table = Arc::new(ListingTable::try_new(config)?); session.register_table(table_ref, listing_table)?; diff --git a/vortex-datafusion/src/convert/scalars.rs b/vortex-datafusion/src/convert/scalars.rs index ab458fec67b..4e684fde66f 100644 --- a/vortex-datafusion/src/convert/scalars.rs +++ b/vortex-datafusion/src/convert/scalars.rs @@ -50,27 +50,55 @@ impl TryToDataFusion for Scalar { let precision = decimal_type.precision(); let scale = decimal_type.scale(); - if precision <= i128::MAX_PRECISION { + if precision <= i32::MAX_PRECISION { + match dscalar.decimal_value() { + None => ScalarValue::Decimal32(None, precision, scale), + Some(value) => match value.cast::() { + Some(v32) => ScalarValue::Decimal32(Some(v32), precision, scale), + None => { + vortex_bail!( + "invalid ScalarValue {value} for decimal with precision {precision}", + ) + } + }, + } + } else if precision <= i64::MAX_PRECISION { + match dscalar.decimal_value() { + None => ScalarValue::Decimal64(None, precision, scale), + Some(value) => match value.cast::() { + Some(v64) => ScalarValue::Decimal64(Some(v64), precision, scale), + None => { + vortex_bail!( + "invalid ScalarValue {value} for decimal with precision {precision}", + ) + } + }, + } + } else if precision <= i128::MAX_PRECISION { match dscalar.decimal_value() { None => ScalarValue::Decimal128(None, precision, scale), - Some(DecimalValue::I128(v128)) => { - ScalarValue::Decimal128(Some(v128), precision, scale) - } - _ => vortex_bail!( - "invalid ScalarValue for decimal with precision {}", - precision - ), + Some(value) => match value.cast::() { + Some(v128) => ScalarValue::Decimal128(Some(v128), precision, scale), + None => { + vortex_bail!( + "invalid ScalarValue {value} for decimal with precision {precision}", + ) + } + }, } } else { match dscalar.decimal_value() { None => ScalarValue::Decimal256(None, precision, scale), - Some(DecimalValue::I256(v256)) => { - ScalarValue::Decimal256(Some(v256.into()), precision, scale) - } - _ => vortex_bail!( - "invalid ScalarValue for decimal with precision {}", - precision - ), + Some(value) => match value.cast::() { + Some(v256) => { + ScalarValue::Decimal256(Some(v256.into()), precision, scale) + } + None => { + vortex_bail!( + "invalid ScalarValue {value} for decimal with precision {precision}", + ) + } + }, } } } @@ -220,6 +248,32 @@ impl FromDataFusion for Scalar { .unwrap_or_else(vortex::scalar::ScalarValue::null), ) } + ScalarValue::Decimal32(decimal, precision, scale) => { + let decimal_dtype = DecimalDType::new(*precision, *scale); + let nullable = Nullability::Nullable; + if let Some(value) = decimal { + Scalar::decimal( + DecimalValue::I32(*value), + decimal_dtype, + Nullability::Nullable, + ) + } else { + Scalar::null(DType::Decimal(decimal_dtype, nullable)) + } + } + ScalarValue::Decimal64(decimal, precision, scale) => { + let decimal_dtype = DecimalDType::new(*precision, *scale); + let nullable = Nullability::Nullable; + if let Some(value) = decimal { + Scalar::decimal( + DecimalValue::I64(*value), + decimal_dtype, + Nullability::Nullable, + ) + } else { + Scalar::null(DType::Decimal(decimal_dtype, nullable)) + } + } ScalarValue::Decimal128(decimal, precision, scale) => { let decimal_dtype = DecimalDType::new(*precision, *scale); let nullable = Nullability::Nullable; @@ -366,17 +420,41 @@ mod tests { } #[rstest] + #[case::decimal32_some( + Scalar::decimal( + DecimalValue::I32(1234), + DecimalDType::new(5, 2), + Nullability::NonNullable + ), + ScalarValue::Decimal32(Some(1234), 5, 2) + )] + #[case::decimal32_null( + Scalar::null(DType::Decimal(DecimalDType::new(5, 2), Nullability::Nullable)), + ScalarValue::Decimal32(None, 5, 2) + )] + #[case::decimal64_some( + Scalar::decimal( + DecimalValue::I64(12345), + DecimalDType::new(10, 2), + Nullability::NonNullable + ), + ScalarValue::Decimal64(Some(12345), 10, 2) + )] + #[case::decimal64_null( + Scalar::null(DType::Decimal(DecimalDType::new(10, 2), Nullability::Nullable)), + ScalarValue::Decimal64(None, 10, 2) + )] #[case::decimal128_some( Scalar::decimal( DecimalValue::I128(12345), - DecimalDType::new(10, 2), + DecimalDType::new(20, 2), Nullability::NonNullable ), - ScalarValue::Decimal128(Some(12345), 10, 2) + ScalarValue::Decimal128(Some(12345), 20, 2) )] #[case::decimal128_null( - Scalar::null(DType::Decimal(DecimalDType::new(10, 2), Nullability::Nullable)), - ScalarValue::Decimal128(None, 10, 2) + Scalar::null(DType::Decimal(DecimalDType::new(20, 2), Nullability::Nullable)), + ScalarValue::Decimal128(None, 20, 2) )] #[case::decimal256_some( Scalar::decimal( @@ -575,8 +653,16 @@ mod tests { ScalarValue::Binary(None) )] #[case::null_decimal128( + Scalar::null(DType::Decimal(DecimalDType::new(20, 2), Nullability::Nullable)), + ScalarValue::Decimal128(None, 20, 2) + )] + #[case::null_decimal64( Scalar::null(DType::Decimal(DecimalDType::new(10, 2), Nullability::Nullable)), - ScalarValue::Decimal128(None, 10, 2) + ScalarValue::Decimal64(None, 10, 2) + )] + #[case::null_decimal32( + Scalar::null(DType::Decimal(DecimalDType::new(5, 2), Nullability::Nullable)), + ScalarValue::Decimal32(None, 5, 2) )] fn test_null_handling(#[case] vortex_null: Scalar, #[case] expected_df_null: ScalarValue) { // Test Vortex -> DataFusion diff --git a/vortex-datafusion/src/persistent/mod.rs b/vortex-datafusion/src/persistent/mod.rs index 3099c3f4762..b1b1e5f727f 100644 --- a/vortex-datafusion/src/persistent/mod.rs +++ b/vortex-datafusion/src/persistent/mod.rs @@ -254,8 +254,9 @@ mod tests { ┌───────────────────────────┐ │ SortPreservingMergeExec │ │ -------------------- │ - │ c1 ASC NULLS LASTlimit: │ - │ 3 │ + │ c1 ASC NULLS LAST │ + │ │ + │ limit: 3 │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ │ DataSourceExec │ diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index 5d836a27153..84c4e84128b 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -14,7 +14,7 @@ use datafusion_common::Result as DFResult; use datafusion_common::arrow::array::RecordBatch; use datafusion_datasource::FileRange; use datafusion_datasource::PartitionedFile; -use datafusion_datasource::file_meta::FileMeta; +use datafusion_datasource::TableSchema; use datafusion_datasource::file_stream::FileOpenFuture; use datafusion_datasource::file_stream::FileOpener; use datafusion_datasource::schema_adapter::SchemaAdapterFactory; @@ -63,12 +63,8 @@ pub(crate) struct VortexOpener { pub file_pruning_predicate: Option, pub expr_adapter_factory: Option>, pub schema_adapter_factory: Arc, - /// Hive-style partitioning columns - pub partition_fields: Vec>, + pub table_schema: TableSchema, pub file_cache: VortexFileCache, - /// This is the table's schema without partition columns. It might be different than - /// the physical schema, and the stream's type will be a projection of it. - pub logical_schema: SchemaRef, pub batch_size: usize, pub limit: Option, pub metrics: VortexMetrics, @@ -159,16 +155,15 @@ fn compute_logical_file_schema( } impl FileOpener for VortexOpener { - fn open(&self, file_meta: FileMeta, file: PartitionedFile) -> DFResult { + fn open(&self, file: PartitionedFile) -> DFResult { let session = self.session.clone(); let object_store = self.object_store.clone(); let projection = self.projection.clone(); let mut filter = self.filter.clone(); let file_pruning_predicate = self.file_pruning_predicate.clone(); let expr_adapter_factory = self.expr_adapter_factory.clone(); - let partition_fields = self.partition_fields.clone(); let file_cache = self.file_cache.clone(); - let logical_schema = self.logical_schema.clone(); + let table_schema = self.table_schema.clone(); let batch_size = self.batch_size; let limit = self.limit; let metrics = self.metrics.clone(); @@ -176,15 +171,15 @@ impl FileOpener for VortexOpener { let has_output_ordering = self.has_output_ordering; let projected_schema = match projection.as_ref() { - None => logical_schema.clone(), - Some(indices) => Arc::new(logical_schema.project(indices)?), + None => table_schema.file_schema().clone(), + Some(indices) => Arc::new(table_schema.file_schema().project(indices)?), }; - let mut predicate_file_schema = logical_schema.clone(); + let mut predicate_file_schema = table_schema.file_schema().clone(); let schema_adapter = self .schema_adapter_factory - .create(projected_schema, logical_schema.clone()); + .create(projected_schema, table_schema.table_schema().clone()); Ok(async move { // Create FilePruner when we have a predicate and either dynamic expressions @@ -200,8 +195,8 @@ impl FileOpener for VortexOpener { (is_dynamic_physical_expr(&predicate) | file.has_statistics()).then_some( FilePruner::new( predicate.clone(), - &logical_schema, - partition_fields.clone(), + table_schema.file_schema(), + table_schema.table_partition_cols().clone(), file.clone(), Count::default(), )?, @@ -220,7 +215,7 @@ impl FileOpener for VortexOpener { } let vxf = file_cache - .try_get(&file_meta.object_meta, object_store) + .try_get(&file.object_meta, object_store) .await .map_err(|e| { DataFusionError::Execution(format!("Failed to open Vortex file {e}")) @@ -233,10 +228,11 @@ impl FileOpener for VortexOpener { // Compute the logical file schema by merging physical file types with logical table types. // This schema has the same field names as logical_schema, but with physical types from the file. let logical_file_schema = - compute_logical_file_schema(&physical_file_schema, &logical_schema); + compute_logical_file_schema(&physical_file_schema, table_schema.file_schema()); if let Some(expr_adapter_factory) = expr_adapter_factory { - let partition_values = partition_fields + let partition_values = table_schema + .table_partition_cols() .iter() .cloned() .zip(file.partition_values) @@ -277,7 +273,7 @@ impl FileOpener for VortexOpener { let projection_expr = select(fields, root()); // We share our layout readers with others partitions in the scan, so we can only need to read each layout in each file once. - let layout_reader = match layout_reader.entry(file_meta.object_meta.location.clone()) { + let layout_reader = match layout_reader.entry(file.object_meta.location.clone()) { Entry::Occupied(mut occupied_entry) => { if let Some(reader) = occupied_entry.get().upgrade() { log::trace!("reusing layout reader for {}", occupied_entry.key()); @@ -305,10 +301,10 @@ impl FileOpener for VortexOpener { }; let mut scan_builder = ScanBuilder::new(session, layout_reader); - if let Some(file_range) = file_meta.range { + if let Some(file_range) = file.range { scan_builder = apply_byte_range( file_range, - file_meta.object_meta.size, + file.object_meta.size, vxf.row_count(), scan_builder, ); @@ -366,7 +362,7 @@ impl FileOpener for VortexOpener { .map_err(move |e: VortexError| { ArrowError::ExternalError(Box::new(e.with_context(format!( "Failed to read Vortex file: {}", - file_meta.object_meta.location + file.object_meta.location )))) }) .try_flatten() @@ -412,13 +408,13 @@ mod tests { use std::sync::LazyLock; use arrow_schema::Fields; - use chrono::Utc; use datafusion::arrow::array::RecordBatch; use datafusion::arrow::array::StringArray; use datafusion::arrow::array::StructArray; use datafusion::arrow::datatypes::DataType; use datafusion::arrow::datatypes::Schema; use datafusion::arrow::util::display::FormatOptions; + use datafusion::arrow::util::pretty::pretty_format_batches_with_options; use datafusion::common::record_batch; use datafusion::datasource::schema_adapter::DefaultSchemaAdapterFactory; use datafusion::logical_expr::col; @@ -428,7 +424,6 @@ mod tests { use datafusion::scalar::ScalarValue; use insta::assert_snapshot; use itertools::Itertools; - use object_store::ObjectMeta; use object_store::memory::InMemory; use rstest::rstest; use vortex::VortexSessionDefault; @@ -502,21 +497,6 @@ mod tests { Ok(summary.size()) } - fn make_meta(path: &str, data_size: u64) -> FileMeta { - FileMeta { - object_meta: ObjectMeta { - location: Path::from(path), - last_modified: Utc::now(), - size: data_size, - e_tag: None, - version: None, - }, - range: None, - extensions: None, - metadata_size_hint: None, - } - } - #[rstest] #[case(Some(Arc::new(DefaultPhysicalExprAdapterFactory) as _), (1, 3), (0, 0))] // If we don't have a physical expr adapter, we just drop filters on partition values @@ -537,10 +517,10 @@ mod tests { let mut file = PartitionedFile::new(file_path.to_string(), data_size); file.partition_values = vec![ScalarValue::Int32(Some(1))]; - let table_schema = Arc::new(Schema::new(vec![ - Field::new("part", DataType::Int32, false), - Field::new("a", DataType::Int32, false), - ])); + let table_schema = TableSchema::new( + file_schema.clone(), + vec![Arc::new(Field::new("part", DataType::Int32, false))], + ); let make_opener = |filter| VortexOpener { session: SESSION.clone(), @@ -550,9 +530,8 @@ mod tests { file_pruning_predicate: None, expr_adapter_factory: expr_adapter_factory.clone(), schema_adapter_factory: Arc::new(DefaultSchemaAdapterFactory), - partition_fields: vec![Arc::new(Field::new("part", DataType::Int32, false))], file_cache: VortexFileCache::new(1, 1, SESSION.clone()), - logical_schema: file_schema.clone(), + table_schema: table_schema.clone(), batch_size: 100, limit: None, metrics: Default::default(), @@ -562,14 +541,10 @@ mod tests { // filter matches partition value let filter = col("part").eq(lit(1)); - let filter = logical2physical(&filter, table_schema.as_ref()); + let filter = logical2physical(&filter, table_schema.table_schema()); let opener = make_opener(filter); - let stream = opener - .open(make_meta(file_path, data_size), file.clone()) - .unwrap() - .await - .unwrap(); + let stream = opener.open(file.clone()).unwrap().await.unwrap(); let data = stream.try_collect::>().await?; let num_batches = data.len(); @@ -579,14 +554,10 @@ mod tests { // filter doesn't matches partition value let filter = col("part").eq(lit(2)); - let filter = logical2physical(&filter, table_schema.as_ref()); + let filter = logical2physical(&filter, table_schema.table_schema()); let opener = make_opener(filter); - let stream = opener - .open(make_meta(file_path, data_size), file.clone()) - .unwrap() - .await - .unwrap(); + let stream = opener.open(file.clone()).unwrap().await.unwrap(); let data = stream.try_collect::>().await?; let num_batches = data.len(); @@ -607,21 +578,30 @@ mod tests { async fn test_open_files_different_table_schema( #[case] expr_adapter_factory: Option>, ) -> anyhow::Result<()> { - use datafusion::arrow::util::pretty::pretty_format_batches_with_options; - let object_store = Arc::new(InMemory::new()) as Arc; - let file1_path = "/path/file1.vortex"; - let batch1 = record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3)])).unwrap(); - let data_size1 = write_arrow_to_vortex(object_store.clone(), file1_path, batch1).await?; - let file1 = PartitionedFile::new(file1_path.to_string(), data_size1); - let file2_path = "/path/file2.vortex"; - let batch2 = record_batch!(("a", Int16, vec![Some(-1), Some(-2), Some(-3)])).unwrap(); - let data_size2 = write_arrow_to_vortex(object_store.clone(), file2_path, batch2).await?; - let file2 = PartitionedFile::new(file1_path.to_string(), data_size1); + let file1 = { + let file1_path = "/path/file1.vortex"; + let batch1 = record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3)])).unwrap(); + let data_size1 = + write_arrow_to_vortex(object_store.clone(), file1_path, batch1).await?; + PartitionedFile::new(file1_path.to_string(), data_size1) + }; + + let file2 = { + let file2_path = "/path/file2.vortex"; + let batch2 = record_batch!(("a", Int16, vec![Some(-1), Some(-2), Some(-3)])).unwrap(); + let data_size2 = + write_arrow_to_vortex(object_store.clone(), file2_path, batch2).await?; + PartitionedFile::new(file2_path.to_string(), data_size2) + }; // Table schema has can accommodate both files - let table_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])); + let table_schema = TableSchema::from_file_schema(Arc::new(Schema::new(vec![Field::new( + "a", + DataType::Int32, + true, + )]))); let make_opener = |filter| VortexOpener { session: SESSION.clone(), @@ -631,9 +611,8 @@ mod tests { file_pruning_predicate: None, expr_adapter_factory: expr_adapter_factory.clone(), schema_adapter_factory: Arc::new(DefaultSchemaAdapterFactory), - partition_fields: vec![], file_cache: VortexFileCache::new(1, 1, SESSION.clone()), - logical_schema: table_schema.clone(), + table_schema: table_schema.clone(), batch_size: 100, limit: None, metrics: Default::default(), @@ -642,12 +621,10 @@ mod tests { }; let filter = col("a").lt(lit(100_i32)); - let filter = logical2physical(&filter, table_schema.as_ref()); + let filter = logical2physical(&filter, table_schema.table_schema()); let opener1 = make_opener(filter.clone()); - let stream = opener1 - .open(make_meta(file1_path, data_size1), file1)? - .await?; + let stream = opener1.open(file1)?.await?; let format_opts = FormatOptions::new().with_types_info(true); @@ -664,9 +641,7 @@ mod tests { "); let opener2 = make_opener(filter.clone()); - let stream = opener2 - .open(make_meta(file2_path, data_size2), file2)? - .await?; + let stream = opener2.open(file2)?.await?; let data = stream.try_collect::>().await?; assert_snapshot!(pretty_format_batches_with_options(&data, &format_opts)?.to_string(), @r" @@ -719,9 +694,8 @@ mod tests { file_pruning_predicate: None, expr_adapter_factory: Some(Arc::new(DefaultPhysicalExprAdapterFactory) as _), schema_adapter_factory: Arc::new(DefaultSchemaAdapterFactory), - partition_fields: vec![], file_cache: VortexFileCache::new(1, 1, SESSION.clone()), - logical_schema: table_schema.clone(), + table_schema: TableSchema::from_file_schema(table_schema.clone()), batch_size: 100, limit: None, metrics: Default::default(), @@ -730,7 +704,7 @@ mod tests { }; // The opener should successfully open the file and reorder columns - let stream = opener.open(make_meta(file_path, data_size), file)?.await?; + let stream = opener.open(file)?.await?; let format_opts = FormatOptions::new().with_types_info(true); let data = stream.try_collect::>().await?; @@ -781,7 +755,7 @@ mod tests { let data_size = write_arrow_to_vortex(object_store.clone(), file_path, batch).await?; // Table schema has an extra utf8 field. - let table_schema = Arc::new(Schema::new(vec![Field::new( + let table_schema = TableSchema::from_file_schema(Arc::new(Schema::new(vec![Field::new( "my_struct", DataType::Struct(Fields::from(vec![ Field::new( @@ -797,7 +771,7 @@ mod tests { Field::new("field3", DataType::Utf8, true), ])), true, - )])); + )]))); let opener = VortexOpener { session: SESSION.clone(), @@ -805,14 +779,13 @@ mod tests { projection: None, filter: Some(logical2physical( &col("my_struct").is_not_null(), - &table_schema, + table_schema.table_schema(), )), file_pruning_predicate: None, expr_adapter_factory: Some(Arc::new(DefaultPhysicalExprAdapterFactory) as _), schema_adapter_factory: Arc::new(DefaultSchemaAdapterFactory), - partition_fields: vec![], file_cache: VortexFileCache::new(1, 1, SESSION.clone()), - logical_schema: table_schema, + table_schema: table_schema.clone(), batch_size: 100, limit: None, metrics: Default::default(), @@ -823,10 +796,7 @@ mod tests { // The opener should be able to open the file with a filter on the // struct column. let data = opener - .open( - make_meta(file_path, data_size), - PartitionedFile::new(file_path.to_string(), data_size), - )? + .open(PartitionedFile::new(file_path.to_string(), data_size))? .await? .try_collect::>() .await?; @@ -856,15 +826,18 @@ mod tests { // Table schema has columns in DIFFERENT order: c, a, b // and different types that require casting (Utf8 -> Dictionary) - let table_schema = Arc::new(Schema::new(vec![ - Field::new("c", DataType::Int32, true), - Field::new("a", DataType::Int32, true), - Field::new( - "b", - DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)), - true, - ), - ])); + let table_schema = TableSchema::new( + Arc::new(Schema::new(vec![ + Field::new("c", DataType::Int32, true), + Field::new("a", DataType::Int32, true), + Field::new( + "b", + DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)), + true, + ), + ])), + vec![], + ); // Project columns [0, 2] from table schema, which should give us: c, b // Before the fix, the schema adapter would get confused about which fields @@ -879,9 +852,8 @@ mod tests { file_pruning_predicate: None, expr_adapter_factory: Some(Arc::new(DefaultPhysicalExprAdapterFactory) as _), schema_adapter_factory: Arc::new(DefaultSchemaAdapterFactory), - partition_fields: vec![], file_cache: VortexFileCache::new(1, 1, SESSION.clone()), - logical_schema: table_schema.clone(), + table_schema: table_schema.clone(), batch_size: 100, limit: None, metrics: Default::default(), @@ -891,10 +863,7 @@ mod tests { // This should succeed and return the correctly projected and cast data let data = opener - .open( - make_meta(file_path, data_size), - PartitionedFile::new(file_path.to_string(), data_size), - )? + .open(PartitionedFile::new(file_path.to_string(), data_size))? .await? .try_collect::>() .await?; diff --git a/vortex-datafusion/src/persistent/source.rs b/vortex-datafusion/src/persistent/source.rs index d42436eb652..fe09c5313ab 100644 --- a/vortex-datafusion/src/persistent/source.rs +++ b/vortex-datafusion/src/persistent/source.rs @@ -6,10 +6,10 @@ use std::fmt::Formatter; use std::sync::Arc; use std::sync::Weak; -use arrow_schema::SchemaRef; use datafusion_common::Result as DFResult; use datafusion_common::Statistics; use datafusion_common::config::ConfigOptions; +use datafusion_datasource::TableSchema; use datafusion_datasource::file::FileSource; use datafusion_datasource::file_scan_config::FileScanConfig; use datafusion_datasource::file_stream::FileOpener; @@ -55,8 +55,7 @@ pub struct VortexSource { pub(crate) vortex_predicate: Option, pub(crate) batch_size: Option, pub(crate) projected_statistics: Option, - /// This is the file schema the table expects, which is the table's schema without partition columns, and **not** the file's physical schema. - pub(crate) arrow_file_schema: Option, + pub(crate) table_schema: Option, pub(crate) schema_adapter_factory: Option>, pub(crate) expr_adapter_factory: Option>, _unused_df_metrics: ExecutionPlanMetricsSet, @@ -75,7 +74,7 @@ impl VortexSource { vortex_predicate: None, batch_size: None, projected_statistics: None, - arrow_file_schema: None, + table_schema: None, schema_adapter_factory: None, expr_adapter_factory: None, _unused_df_metrics: Default::default(), @@ -140,6 +139,8 @@ impl FileSource for VortexSource { let projection = base_config.file_column_projection_indices().map(Arc::from); + let table_schema = base_config.table_schema.clone(); + let opener = VortexOpener { session: self.session.clone(), object_store, @@ -148,8 +149,7 @@ impl FileSource for VortexSource { file_pruning_predicate: self.full_predicate.clone(), expr_adapter_factory, schema_adapter_factory, - partition_fields: base_config.table_partition_cols.clone(), - logical_schema: base_config.file_schema.clone(), + table_schema, file_cache: self.file_cache.clone(), batch_size, limit: base_config.limit, @@ -171,9 +171,9 @@ impl FileSource for VortexSource { Arc::new(source) } - fn with_schema(&self, schema: SchemaRef) -> Arc { + fn with_schema(&self, schema: TableSchema) -> Arc { let mut source = self.clone(); - source.arrow_file_schema = Some(schema); + source.table_schema = Some(schema); Arc::new(source) } @@ -240,7 +240,7 @@ impl FileSource for VortexSource { )); } - let Some(schema) = self.arrow_file_schema.as_ref() else { + let Some(table_schema) = self.table_schema.as_ref() else { return Ok(FilterPushdownPropagation::with_parent_pushdown_result( vec![PushedDown::No; filters.len()], )); @@ -260,7 +260,7 @@ impl FileSource for VortexSource { let supported_filters = filters .into_iter() .map(|expr| { - if can_be_pushed_down(&expr, schema) { + if can_be_pushed_down(&expr, table_schema.file_schema()) { PushedDownPredicate::supported(expr) } else { PushedDownPredicate::unsupported(expr) diff --git a/vortex-dtype/src/arrow.rs b/vortex-dtype/src/arrow.rs index 75430496af1..11935316f27 100644 --- a/vortex-dtype/src/arrow.rs +++ b/vortex-dtype/src/arrow.rs @@ -214,13 +214,12 @@ impl DType { let scale = dt.scale(); match precision { - // This code is commented out until DataFusion improves its support for smaller decimals. - // // DECIMAL32_MAX_PRECISION - // 0..=9 => DataType::Decimal32(precision, scale), - // // DECIMAL64_MAX_PRECISION - // 10..=18 => DataType::Decimal64(precision, scale), + // DECIMAL32_MAX_PRECISION + 0..=9 => DataType::Decimal32(precision, scale), + // DECIMAL64_MAX_PRECISION + 10..=18 => DataType::Decimal64(precision, scale), // DECIMAL128_MAX_PRECISION - 0..=38 => DataType::Decimal128(precision, scale), + 19..=38 => DataType::Decimal128(precision, scale), // DECIMAL256_MAX_PRECISION 39.. => DataType::Decimal256(precision, scale), } From cb25f3c0c08b56f669c15494611ba9b4e5440052 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Tue, 9 Dec 2025 09:17:05 +0000 Subject: [PATCH 2/8] Patch tpchgen Signed-off-by: Adam Gutglick --- Cargo.lock | 31 +- Cargo.toml | 2 - bench-vortex/Cargo.toml | 4 +- java/testfiles/Cargo.lock | 591 +++++++++------------------ wasm-test/Cargo.lock | 820 ++++++++++++++------------------------ 5 files changed, 493 insertions(+), 955 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9ccded7179..64f2af3c92d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4496,7 +4496,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core", ] [[package]] @@ -8859,16 +8859,14 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tpchgen" version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5315a6fb66b84b9dc4ade3ce3d85fc246305c87a8106193e9565b8a45394cbe" +source = "git+https://github.com/clflushopt/tpchgen-rs.git?rev=c38f69b0dc84c4b7edfad349d1a9cd1a134b8018#c38f69b0dc84c4b7edfad349d1a9cd1a134b8018" [[package]] name = "tpchgen-arrow" version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e70da6e86f54ff3524628dc84cb0a9e674ea28e8a8a82fe3839af8c7fd743b" +source = "git+https://github.com/clflushopt/tpchgen-rs.git?rev=c38f69b0dc84c4b7edfad349d1a9cd1a134b8018#c38f69b0dc84c4b7edfad349d1a9cd1a134b8018" dependencies = [ - "arrow 56.2.0", + "arrow 57.0.0", "tpchgen", ] @@ -10192,7 +10190,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ "windows-collections", - "windows-core 0.61.2", + "windows-core", "windows-future", "windows-link 0.1.3", "windows-numerics", @@ -10204,7 +10202,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" dependencies = [ - "windows-core 0.61.2", + "windows-core", ] [[package]] @@ -10220,26 +10218,13 @@ dependencies = [ "windows-strings 0.4.2", ] -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", -] - [[package]] name = "windows-future" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ - "windows-core 0.61.2", + "windows-core", "windows-link 0.1.3", "windows-threading", ] @@ -10284,7 +10269,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" dependencies = [ - "windows-core 0.61.2", + "windows-core", "windows-link 0.1.3", ] diff --git a/Cargo.toml b/Cargo.toml index 16c7510e4d0..8fb170d1106 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -201,8 +201,6 @@ thiserror = "2.0.3" tokio = { version = "1.48" } tokio-stream = "0.1.17" tokio-util = { version = "0.7.16" } -tpchgen = { version = "2.0.1" } -tpchgen-arrow = { version = "2.0.1" } tracing = { version = "0.1.41" } tracing-perfetto = "0.1.5" tracing-subscriber = "0.3.20" diff --git a/bench-vortex/Cargo.toml b/bench-vortex/Cargo.toml index 4bab8b9c093..25b268a5003 100644 --- a/bench-vortex/Cargo.toml +++ b/bench-vortex/Cargo.toml @@ -73,8 +73,8 @@ tempfile = { workspace = true } tokio = { workspace = true, features = ["full"] } tokio-stream = { workspace = true } tokio-util = { workspace = true } -tpchgen = { workspace = true } -tpchgen-arrow = { workspace = true } +tpchgen = { git = "https://github.com/clflushopt/tpchgen-rs.git", rev = "c38f69b0dc84c4b7edfad349d1a9cd1a134b8018" } +tpchgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs.git", rev = "c38f69b0dc84c4b7edfad349d1a9cd1a134b8018" } tracing = { workspace = true } tracing-perfetto = { workspace = true } tracing-subscriber = { workspace = true, features = [ diff --git a/java/testfiles/Cargo.lock b/java/testfiles/Cargo.lock index 255cb58477b..a316c3ccdb1 100644 --- a/java/testfiles/Cargo.lock +++ b/java/testfiles/Cargo.lock @@ -2,21 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "addr2line" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - [[package]] name = "ahash" version = "0.8.12" @@ -25,7 +10,7 @@ checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", "const-random", - "getrandom 0.3.3", + "getrandom 0.3.4", "once_cell", "version_check", "zerocopy", @@ -33,9 +18,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] @@ -75,23 +60,23 @@ checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrow-arith" -version = "56.2.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" +checksum = "0f377dcd19e440174596d83deb49cd724886d91060c07fec4f67014ef9d54049" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "num", + "num-traits", ] [[package]] name = "arrow-array" -version = "56.2.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" +checksum = "a23eaff85a44e9fa914660fb0d0bb00b79c4a3d888b5334adb3ea4330c84f002" dependencies = [ "ahash", "arrow-buffer", @@ -99,38 +84,42 @@ dependencies = [ "arrow-schema", "chrono", "half", - "hashbrown 0.16.0", - "num", + "hashbrown 0.16.1", + "num-complex", + "num-integer", + "num-traits", ] [[package]] name = "arrow-buffer" -version = "56.2.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" +checksum = "a2819d893750cb3380ab31ebdc8c68874dd4429f90fd09180f3c93538bd21626" dependencies = [ "bytes", "half", - "num", + "num-bigint", + "num-traits", ] [[package]] name = "arrow-data" -version = "56.2.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" +checksum = "05738f3d42cb922b9096f7786f606fcb8669260c2640df8490533bb2fa38c9d3" dependencies = [ "arrow-buffer", "arrow-schema", "half", - "num", + "num-integer", + "num-traits", ] [[package]] name = "arrow-ord" -version = "56.2.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" +checksum = "cbc94fc7adec5d1ba9e8cd1b1e8d6f72423b33fe978bf1f46d970fafab787521" dependencies = [ "arrow-array", "arrow-buffer", @@ -141,32 +130,32 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "56.2.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" +checksum = "d27609cd7dd45f006abae27995c2729ef6f4b9361cde1ddd019dc31a5aa017e0" dependencies = [ "bitflags", ] [[package]] name = "arrow-select" -version = "56.2.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" +checksum = "ae980d021879ea119dd6e2a13912d81e64abed372d53163e804dfe84639d8010" dependencies = [ "ahash", "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", - "num", + "num-traits", ] [[package]] name = "arrow-string" -version = "56.2.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" +checksum = "cf35e8ef49dcf0c5f6d175edee6b8af7b45611805333129c541a8b89a0fc0534" dependencies = [ "arrow-array", "arrow-buffer", @@ -174,7 +163,7 @@ dependencies = [ "arrow-schema", "arrow-select", "memchr", - "num", + "num-traits", "regex", "regex-syntax", ] @@ -244,7 +233,7 @@ dependencies = [ "polling", "rustix", "slab", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -302,7 +291,7 @@ dependencies = [ "rustix", "signal-hook-registry", "slab", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -324,7 +313,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -341,7 +330,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -356,21 +345,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "backtrace" -version = "0.3.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-link", -] - [[package]] name = "better_io" version = "0.1.0" @@ -385,9 +359,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.9.4" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" [[package]] name = "bitvec" @@ -422,15 +396,15 @@ checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "bytes" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" [[package]] name = "cc" -version = "1.2.41" +version = "1.2.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7" +checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215" dependencies = [ "find-msvc-tools", "jobserver", @@ -440,9 +414,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chrono" @@ -528,9 +502,9 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "cudarc" -version = "0.17.8" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf99ab37ee7072d64d906aa2dada9a3422f1d975cdf8c8055a573bc84897ed8" +checksum = "ef0cfc5e22a6b6f7d04ee45b0151232ca236ede8ca3534210fd4072bdead0d60" dependencies = [ "half", "libloading", @@ -558,7 +532,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -590,7 +564,7 @@ checksum = "685adfa4d6f3d765a26bc5dbc936577de9abf756c1feeb3089b01dd395034842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -610,7 +584,7 @@ checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -626,7 +600,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -652,9 +626,9 @@ dependencies = [ [[package]] name = "exponential-decay-histogram" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "988360e80225a42d5e2f78fcb90ef7d54d1e4394d335476b5465d34942426776" +checksum = "d7962d7e9baab6ea05af175491fa6a8441f3bf461558037622142573d98dd6d6" dependencies = [ "ordered-float 5.1.0", "rand", @@ -710,9 +684,9 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "find-msvc-tools" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" [[package]] name = "flatbuffers" @@ -741,9 +715,9 @@ dependencies = [ [[package]] name = "fsst-rs" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "738dac3bb05f3cbad316400cbf148107fe0183ef42f971d8636f4014e3c8f013" +checksum = "561f2458a3407836ab8f1acc9113b8cda91b9d6378ba8dad13b2fe1a1d3af5ce" [[package]] name = "funty" @@ -820,7 +794,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -861,34 +835,28 @@ checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "libc", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", ] [[package]] name = "getrandom" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "js-sys", "libc", "r-efi", - "wasi 0.14.7+wasi-0.2.4", + "wasip2", "wasm-bindgen", ] -[[package]] -name = "gimli" -version = "0.32.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" - [[package]] name = "half" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e54c115d4f30f52c67202f079c5f9d8b49db4691f460fdb0b4c2e838261b2ba5" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "cfg-if", "crunchy", @@ -912,9 +880,9 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "hashbrown" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", @@ -962,9 +930,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" dependencies = [ "displaydoc", "potential_utf", @@ -975,9 +943,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" dependencies = [ "displaydoc", "litemap", @@ -988,11 +956,10 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" dependencies = [ - "displaydoc", "icu_collections", "icu_normalizer_data", "icu_properties", @@ -1003,42 +970,38 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" [[package]] name = "icu_properties" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99" dependencies = [ - "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", "icu_provider", - "potential_utf", "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" +checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899" [[package]] name = "icu_provider" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" dependencies = [ "displaydoc", "icu_locale_core", - "stable_deref_trait", - "tinystr", "writeable", "yoke", "zerofrom", @@ -1076,17 +1039,6 @@ dependencies = [ "rustversion", ] -[[package]] -name = "io-uring" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" -dependencies = [ - "bitflags", - "cfg-if", - "libc", -] - [[package]] name = "itertools" version = "0.14.0" @@ -1098,28 +1050,28 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49" +checksum = "49cce2b81f2098e7e3efc35bc2e0a6b7abec9d34128283d7a26fa8f32a6dbb35" dependencies = [ "jiff-static", "jiff-tzdb-platform", "log", "portable-atomic", "portable-atomic-util", - "serde", - "windows-sys 0.59.0", + "serde_core", + "windows-sys", ] [[package]] name = "jiff-static" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" +checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -1143,15 +1095,15 @@ version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", "libc", ] [[package]] name = "js-sys" -version = "0.3.81" +version = "0.3.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" +checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" dependencies = [ "once_cell", "wasm-bindgen", @@ -1194,9 +1146,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.177" +version = "0.2.178" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" [[package]] name = "libloading" @@ -1222,9 +1174,9 @@ checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "litemap" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" [[package]] name = "lock_api" @@ -1237,9 +1189,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.28" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "macro_rules_attribute" @@ -1263,26 +1215,6 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", -] - -[[package]] -name = "mio" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" -dependencies = [ - "libc", - "wasi 0.11.1+wasi-snapshot-preview1", - "windows-sys 0.59.0", -] - [[package]] name = "moka" version = "0.12.11" @@ -1322,7 +1254,7 @@ checksum = "b093064383341eb3271f42e381cb8f10a01459478446953953c75d24bd339fc0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", "target-features", ] @@ -1353,20 +1285,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - [[package]] name = "num-bigint" version = "0.4.6" @@ -1395,28 +1313,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-iter" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -1429,9 +1325,9 @@ dependencies = [ [[package]] name = "num_enum" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a973b4e44ce6cad84ce69d797acf9a044532e4184c4f267913d1b546a0727b7a" +checksum = "b1207a7e20ad57b847bbddc6776b968420d38292bbfe2089accff5e19e82454c" dependencies = [ "num_enum_derive", "rustversion", @@ -1439,22 +1335,13 @@ dependencies = [ [[package]] name = "num_enum_derive" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e878c846a8abae00dd069496dbe8751b16ac1c3d6bd2a7283a938e8228f90d" +checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", -] - -[[package]] -name = "object" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" -dependencies = [ - "memchr", + "syn 2.0.111", ] [[package]] @@ -1527,9 +1414,9 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "pco" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab068b64f2c6f074cbdcafc80ebd83a27da92a3848deba2fabc21eba6691fc65" +checksum = "daea1197f2969fab4d5c6620eade5d46c98a8e9b04ad2bc3725fc5dfc4eb8a49" dependencies = [ "better_io", "dtype_dispatch", @@ -1583,7 +1470,7 @@ dependencies = [ "hermit-abi", "pin-project-lite", "rustix", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -1609,9 +1496,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" dependencies = [ "zerovec", ] @@ -1627,9 +1514,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.101" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" dependencies = [ "unicode-ident", ] @@ -1654,7 +1541,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -1668,9 +1555,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.41" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" dependencies = [ "proc-macro2", ] @@ -1719,7 +1606,7 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", ] [[package]] @@ -1752,9 +1639,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.1" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a52d8d02cacdb176ef4678de6c052efb4b3da14b78e4db683a4252762be5433" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", @@ -1764,9 +1651,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "722166aa0d7438abbaa4d5cc2c649dac844e8c56d82fb3d33e9c34b5cd268fc6" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", @@ -1775,15 +1662,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3160422bbd54dd5ecfdca71e5fd59b7b8fe2b1697ab2baf64f6d05dcc66d298" - -[[package]] -name = "rustc-demangle" -version = "0.1.26" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "rustc-hash" @@ -1810,7 +1691,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -1874,7 +1755,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -1885,9 +1766,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" -version = "1.4.6" +version = "1.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" +checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" dependencies = [ "libc", ] @@ -1958,9 +1839,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.106" +version = "2.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" dependencies = [ "proc-macro2", "quote", @@ -1975,7 +1856,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -2021,9 +1902,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" dependencies = [ "displaydoc", "zerovec", @@ -2031,24 +1912,19 @@ dependencies = [ [[package]] name = "tokio" -version = "1.47.1" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" dependencies = [ - "backtrace", "bytes", - "io-uring", - "libc", - "mio", "pin-project-lite", - "slab", ] [[package]] name = "tracing" -version = "0.1.41" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" dependencies = [ "pin-project-lite", "tracing-attributes", @@ -2057,29 +1933,29 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] name = "tracing-core" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" dependencies = [ "once_cell", ] [[package]] name = "unicode-ident" -version = "1.0.19" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "url" @@ -2101,11 +1977,11 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.18.1" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", "js-sys", "wasm-bindgen", ] @@ -2168,7 +2044,9 @@ dependencies = [ "vortex-fastlanes", "vortex-mask", "vortex-scalar", + "vortex-session", "vortex-utils", + "vortex-vector", ] [[package]] @@ -2191,7 +2069,7 @@ dependencies = [ "enum-map", "flatbuffers", "futures", - "getrandom 0.3.3", + "getrandom 0.3.4", "humansize", "inventory", "itertools", @@ -2227,7 +2105,7 @@ dependencies = [ name = "vortex-btrblocks" version = "0.1.0" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", "itertools", "log", "num-traits", @@ -2285,7 +2163,10 @@ dependencies = [ "arrow-array", "arrow-buffer", "arrow-schema", + "log", + "multiversion", "num-traits", + "paste", "vortex-buffer", "vortex-dtype", "vortex-error", @@ -2375,6 +2256,7 @@ dependencies = [ "vortex-error", "vortex-mask", "vortex-scalar", + "vortex-session", "vortex-utils", "vortex-vector", ] @@ -2387,7 +2269,7 @@ dependencies = [ "bytes", "flatbuffers", "futures", - "getrandom 0.3.3", + "getrandom 0.3.4", "itertools", "kanal", "log", @@ -2454,7 +2336,7 @@ dependencies = [ "bytes", "cfg-if", "futures", - "getrandom 0.3.3", + "getrandom 0.3.4", "handle", "kanal", "log", @@ -2542,7 +2424,7 @@ dependencies = [ name = "vortex-metrics" version = "0.1.0" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", "parking_lot", "vortex-session", "witchcraft-metrics", @@ -2603,8 +2485,10 @@ dependencies = [ "vortex-buffer", "vortex-dtype", "vortex-error", + "vortex-mask", "vortex-proto", "vortex-utils", + "vortex-vector", ] [[package]] @@ -2677,7 +2561,7 @@ name = "vortex-utils" version = "0.1.0" dependencies = [ "dashmap", - "hashbrown 0.16.0", + "hashbrown 0.16.1", ] [[package]] @@ -2729,15 +2613,6 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" -[[package]] -name = "wasi" -version = "0.14.7+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" -dependencies = [ - "wasip2", -] - [[package]] name = "wasip2" version = "1.0.1+wasi-0.2.4" @@ -2749,9 +2624,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.104" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" +checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" dependencies = [ "cfg-if", "once_cell", @@ -2760,25 +2635,11 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn 2.0.106", - "wasm-bindgen-shared", -] - [[package]] name = "wasm-bindgen-futures" -version = "0.4.54" +version = "0.4.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e038d41e478cc73bae0ff9b36c60cff1c98b8f38f8d7e8061e79ee63608ac5c" +checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" dependencies = [ "cfg-if", "js-sys", @@ -2789,9 +2650,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.104" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" +checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2799,31 +2660,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.104" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" +checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" dependencies = [ + "bumpalo", "proc-macro2", "quote", - "syn 2.0.106", - "wasm-bindgen-backend", + "syn 2.0.111", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.104" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" +checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.81" +version = "0.3.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9367c417a924a74cae129e6a2ae3b47fabb1f8995595ab474029da749a8be120" +checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" dependencies = [ "js-sys", "wasm-bindgen", @@ -2850,7 +2711,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -2861,7 +2722,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -2888,15 +2749,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -2906,70 +2758,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - [[package]] name = "wit-bindgen" version = "0.46.0" @@ -2991,9 +2779,9 @@ dependencies = [ [[package]] name = "writeable" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" [[package]] name = "wyz" @@ -3006,11 +2794,10 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" dependencies = [ - "serde", "stable_deref_trait", "yoke-derive", "zerofrom", @@ -3018,34 +2805,34 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.27" +version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.27" +version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -3065,15 +2852,15 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", "synstructure", ] [[package]] name = "zerotrie" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" dependencies = [ "displaydoc", "yoke", @@ -3082,9 +2869,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.4" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" dependencies = [ "yoke", "zerofrom", @@ -3093,13 +2880,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] diff --git a/wasm-test/Cargo.lock b/wasm-test/Cargo.lock index e3ec17002e9..a3d499787bd 100644 --- a/wasm-test/Cargo.lock +++ b/wasm-test/Cargo.lock @@ -2,21 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "addr2line" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - [[package]] name = "ahash" version = "0.8.12" @@ -25,7 +10,7 @@ checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", "const-random", - "getrandom 0.3.3", + "getrandom 0.3.4", "once_cell", "version_check", "zerocopy", @@ -33,9 +18,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] @@ -75,23 +60,23 @@ checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrow-arith" -version = "56.2.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" +checksum = "0f377dcd19e440174596d83deb49cd724886d91060c07fec4f67014ef9d54049" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "num", + "num-traits", ] [[package]] name = "arrow-array" -version = "56.2.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" +checksum = "a23eaff85a44e9fa914660fb0d0bb00b79c4a3d888b5334adb3ea4330c84f002" dependencies = [ "ahash", "arrow-buffer", @@ -99,38 +84,42 @@ dependencies = [ "arrow-schema", "chrono", "half", - "hashbrown 0.16.0", - "num", + "hashbrown 0.16.1", + "num-complex", + "num-integer", + "num-traits", ] [[package]] name = "arrow-buffer" -version = "56.2.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" +checksum = "a2819d893750cb3380ab31ebdc8c68874dd4429f90fd09180f3c93538bd21626" dependencies = [ "bytes", "half", - "num", + "num-bigint", + "num-traits", ] [[package]] name = "arrow-data" -version = "56.2.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" +checksum = "05738f3d42cb922b9096f7786f606fcb8669260c2640df8490533bb2fa38c9d3" dependencies = [ "arrow-buffer", "arrow-schema", "half", - "num", + "num-integer", + "num-traits", ] [[package]] name = "arrow-ord" -version = "56.2.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" +checksum = "cbc94fc7adec5d1ba9e8cd1b1e8d6f72423b33fe978bf1f46d970fafab787521" dependencies = [ "arrow-array", "arrow-buffer", @@ -141,32 +130,32 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "56.2.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" +checksum = "d27609cd7dd45f006abae27995c2729ef6f4b9361cde1ddd019dc31a5aa017e0" dependencies = [ "bitflags", ] [[package]] name = "arrow-select" -version = "56.2.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" +checksum = "ae980d021879ea119dd6e2a13912d81e64abed372d53163e804dfe84639d8010" dependencies = [ "ahash", "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", - "num", + "num-traits", ] [[package]] name = "arrow-string" -version = "56.2.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" +checksum = "cf35e8ef49dcf0c5f6d175edee6b8af7b45611805333129c541a8b89a0fc0534" dependencies = [ "arrow-array", "arrow-buffer", @@ -174,7 +163,7 @@ dependencies = [ "arrow-schema", "arrow-select", "memchr", - "num", + "num-traits", "regex", "regex-syntax", ] @@ -244,7 +233,7 @@ dependencies = [ "polling", "rustix", "slab", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -302,7 +291,7 @@ dependencies = [ "rustix", "signal-hook-registry", "slab", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -324,7 +313,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -341,7 +330,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -356,21 +345,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "backtrace" -version = "0.3.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-link", -] - [[package]] name = "better_io" version = "0.1.0" @@ -385,9 +359,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.9.4" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" [[package]] name = "bitvec" @@ -422,15 +396,15 @@ checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "bytes" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" [[package]] name = "cc" -version = "1.2.41" +version = "1.2.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7" +checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215" dependencies = [ "find-msvc-tools", "shlex", @@ -438,9 +412,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chrono" @@ -462,18 +436,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "console" -version = "0.15.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" -dependencies = [ - "encode_unicode", - "libc", - "once_cell", - "windows-sys 0.59.0", -] - [[package]] name = "const-random" version = "0.1.18" @@ -515,16 +477,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - [[package]] name = "crossbeam-epoch" version = "0.9.18" @@ -534,15 +486,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "crossbeam-queue" -version = "0.3.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -555,6 +498,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "cudarc" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef0cfc5e22a6b6f7d04ee45b0151232ca236ede8ca3534210fd4072bdead0d60" +dependencies = [ + "half", + "libloading", +] + [[package]] name = "dashmap" version = "6.1.0" @@ -577,7 +530,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -586,24 +539,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3a5ccdfd6c5e7e2fea9c5cf256f2a08216047fab19c621c3da64e9ae4a1462d" -[[package]] -name = "dyn-hash" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15401da73a9ed8c80e3b2d4dc05fe10e7b72d7243b9f614e516a44fa99986e88" - [[package]] name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" -[[package]] -name = "encode_unicode" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" - [[package]] name = "enum-iterator" version = "2.3.0" @@ -621,7 +562,27 @@ checksum = "685adfa4d6f3d765a26bc5dbc936577de9abf756c1feeb3089b01dd395034842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", +] + +[[package]] +name = "enum-map" +version = "2.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6866f3bfdf8207509a033af1a75a7b08abda06bbaaeae6669323fd5a097df2e9" +dependencies = [ + "enum-map-derive", +] + +[[package]] +name = "enum-map-derive" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", ] [[package]] @@ -637,7 +598,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -663,9 +624,9 @@ dependencies = [ [[package]] name = "exponential-decay-histogram" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "988360e80225a42d5e2f78fcb90ef7d54d1e4394d335476b5465d34942426776" +checksum = "d7962d7e9baab6ea05af175491fa6a8441f3bf461558037622142573d98dd6d6" dependencies = [ "ordered-float 5.1.0", "rand", @@ -702,9 +663,9 @@ dependencies = [ [[package]] name = "fastlanes" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b79094ac813a000b92adff4231a6d291d063dc6312b3b88c154a5ea0a195d0" +checksum = "414cb755aee48ff7b0907995d2949c68c8c17900970076dff6a808e18e592d71" dependencies = [ "arrayref", "const_for", @@ -721,9 +682,9 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "find-msvc-tools" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" [[package]] name = "flatbuffers" @@ -752,9 +713,9 @@ dependencies = [ [[package]] name = "fsst-rs" -version = "0.5.4" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab195789b87bb56fce91b3617e44d36dbba68a4c8d736ef48767187932a5161b" +checksum = "561f2458a3407836ab8f1acc9113b8cda91b9d6378ba8dad13b2fe1a1d3af5ce" [[package]] name = "funty" @@ -831,7 +792,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -872,38 +833,34 @@ checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "libc", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", ] [[package]] name = "getrandom" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "js-sys", "libc", "r-efi", - "wasi 0.14.7+wasi-0.2.4", + "wasip2", "wasm-bindgen", ] -[[package]] -name = "gimli" -version = "0.32.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" - [[package]] name = "half" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e54c115d4f30f52c67202f079c5f9d8b49db4691f460fdb0b4c2e838261b2ba5" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "cfg-if", "crunchy", "num-traits", + "rand", + "rand_distr", "zerocopy", ] @@ -921,9 +878,9 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "hashbrown" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", @@ -971,9 +928,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" dependencies = [ "displaydoc", "potential_utf", @@ -984,9 +941,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" dependencies = [ "displaydoc", "litemap", @@ -997,11 +954,10 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" dependencies = [ - "displaydoc", "icu_collections", "icu_normalizer_data", "icu_properties", @@ -1012,42 +968,38 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" [[package]] name = "icu_properties" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99" dependencies = [ - "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", "icu_provider", - "potential_utf", "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" +checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899" [[package]] name = "icu_provider" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" dependencies = [ "displaydoc", "icu_locale_core", - "stable_deref_trait", - "tinystr", "writeable", "yoke", "zerofrom", @@ -1076,17 +1028,6 @@ dependencies = [ "icu_properties", ] -[[package]] -name = "insta" -version = "1.43.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fdb647ebde000f43b5b53f773c30cf9b0cb4300453208713fa38b2c70935a0" -dependencies = [ - "console", - "once_cell", - "similar", -] - [[package]] name = "inventory" version = "0.3.21" @@ -1096,17 +1037,6 @@ dependencies = [ "rustversion", ] -[[package]] -name = "io-uring" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" -dependencies = [ - "bitflags", - "cfg-if", - "libc", -] - [[package]] name = "itertools" version = "0.14.0" @@ -1118,28 +1048,28 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49" +checksum = "49cce2b81f2098e7e3efc35bc2e0a6b7abec9d34128283d7a26fa8f32a6dbb35" dependencies = [ "jiff-static", "jiff-tzdb-platform", "log", "portable-atomic", "portable-atomic-util", - "serde", - "windows-sys 0.59.0", + "serde_core", + "windows-sys", ] [[package]] name = "jiff-static" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" +checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -1159,9 +1089,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.81" +version = "0.3.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" +checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" dependencies = [ "once_cell", "wasm-bindgen", @@ -1204,9 +1134,19 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.177" +version = "0.2.178" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] [[package]] name = "libm" @@ -1222,9 +1162,9 @@ checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "litemap" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" [[package]] name = "lock_api" @@ -1237,9 +1177,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.28" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "macro_rules_attribute" @@ -1263,26 +1203,6 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", -] - -[[package]] -name = "mio" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" -dependencies = [ - "libc", - "wasi 0.11.1+wasi-snapshot-preview1", - "windows-sys 0.59.0", -] - [[package]] name = "moka" version = "0.12.11" @@ -1322,7 +1242,7 @@ checksum = "b093064383341eb3271f42e381cb8f10a01459478446953953c75d24bd339fc0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", "target-features", ] @@ -1353,20 +1273,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - [[package]] name = "num-bigint" version = "0.4.6" @@ -1395,28 +1301,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-iter" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -1429,9 +1313,9 @@ dependencies = [ [[package]] name = "num_enum" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a973b4e44ce6cad84ce69d797acf9a044532e4184c4f267913d1b546a0727b7a" +checksum = "b1207a7e20ad57b847bbddc6776b968420d38292bbfe2089accff5e19e82454c" dependencies = [ "num_enum_derive", "rustversion", @@ -1439,22 +1323,13 @@ dependencies = [ [[package]] name = "num_enum_derive" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e878c846a8abae00dd069496dbe8751b16ac1c3d6bd2a7283a938e8228f90d" +checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", -] - -[[package]] -name = "object" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" -dependencies = [ - "memchr", + "syn 2.0.111", ] [[package]] @@ -1543,26 +1418,6 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" -[[package]] -name = "pin-project" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - [[package]] name = "pin-project-lite" version = "0.2.16" @@ -1597,7 +1452,7 @@ dependencies = [ "hermit-abi", "pin-project-lite", "rustix", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -1623,9 +1478,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" dependencies = [ "zerovec", ] @@ -1641,9 +1496,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.101" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" dependencies = [ "unicode-ident", ] @@ -1668,7 +1523,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -1682,9 +1537,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.41" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" dependencies = [ "proc-macro2", ] @@ -1733,7 +1588,17 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", +] + +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand", ] [[package]] @@ -1756,9 +1621,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.1" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a52d8d02cacdb176ef4678de6c052efb4b3da14b78e4db683a4252762be5433" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", @@ -1768,9 +1633,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "722166aa0d7438abbaa4d5cc2c649dac844e8c56d82fb3d33e9c34b5cd268fc6" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", @@ -1779,15 +1644,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3160422bbd54dd5ecfdca71e5fd59b7b8fe2b1697ab2baf64f6d05dcc66d298" - -[[package]] -name = "rustc-demangle" -version = "0.1.26" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "rustc-hash" @@ -1814,7 +1673,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -1878,7 +1737,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -1889,9 +1748,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" -version = "1.4.6" +version = "1.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" +checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" dependencies = [ "libc", ] @@ -1902,12 +1761,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" -[[package]] -name = "similar" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" - [[package]] name = "sketches-ddsketch" version = "0.3.0" @@ -1968,9 +1821,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.106" +version = "2.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" dependencies = [ "proc-macro2", "quote", @@ -1985,7 +1838,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -2023,9 +1876,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" dependencies = [ "displaydoc", "zerovec", @@ -2033,24 +1886,19 @@ dependencies = [ [[package]] name = "tokio" -version = "1.47.1" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" dependencies = [ - "backtrace", "bytes", - "io-uring", - "libc", - "mio", "pin-project-lite", - "slab", ] [[package]] name = "tracing" -version = "0.1.41" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" dependencies = [ "pin-project-lite", "tracing-attributes", @@ -2059,29 +1907,29 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] name = "tracing-core" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" dependencies = [ "once_cell", ] [[package]] name = "unicode-ident" -version = "1.0.19" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "url" @@ -2103,11 +1951,11 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.18.1" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", "js-sys", "wasm-bindgen", ] @@ -2122,6 +1970,8 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" name = "vortex" version = "0.1.0" dependencies = [ + "fastlanes", + "rand", "vortex-alp", "vortex-array", "vortex-btrblocks", @@ -2129,10 +1979,8 @@ dependencies = [ "vortex-bytebool", "vortex-datetime-parts", "vortex-decimal-byte-parts", - "vortex-dict", "vortex-dtype", "vortex-error", - "vortex-expr", "vortex-fastlanes", "vortex-flatbuffers", "vortex-fsst", @@ -2147,6 +1995,7 @@ dependencies = [ "vortex-scalar", "vortex-scan", "vortex-sequence", + "vortex-session", "vortex-sparse", "vortex-utils", "vortex-zigzag", @@ -2167,7 +2016,9 @@ dependencies = [ "vortex-fastlanes", "vortex-mask", "vortex-scalar", + "vortex-session", "vortex-utils", + "vortex-vector", ] [[package]] @@ -2187,11 +2038,11 @@ dependencies = [ "bitvec", "cfg-if", "enum-iterator", + "enum-map", "flatbuffers", "futures", - "getrandom 0.3.3", + "getrandom 0.3.4", "humansize", - "insta", "inventory", "itertools", "log", @@ -2200,7 +2051,7 @@ dependencies = [ "num_enum", "parking_lot", "paste", - "pin-project", + "pin-project-lite", "prost", "rand", "rustc-hash", @@ -2208,21 +2059,25 @@ dependencies = [ "static_assertions", "termtree", "vortex-buffer", + "vortex-compute", "vortex-dtype", "vortex-error", "vortex-flatbuffers", + "vortex-io", "vortex-mask", "vortex-metrics", + "vortex-proto", "vortex-scalar", + "vortex-session", "vortex-utils", + "vortex-vector", ] [[package]] name = "vortex-btrblocks" version = "0.1.0" dependencies = [ - "arrow-buffer", - "getrandom 0.3.3", + "getrandom 0.3.4", "itertools", "log", "num-traits", @@ -2233,7 +2088,6 @@ dependencies = [ "vortex-buffer", "vortex-datetime-parts", "vortex-decimal-byte-parts", - "vortex-dict", "vortex-dtype", "vortex-error", "vortex-fastlanes", @@ -2252,7 +2106,9 @@ name = "vortex-buffer" version = "0.1.0" dependencies = [ "arrow-buffer", + "bitvec", "bytes", + "cudarc", "itertools", "num-traits", "simdutf8", @@ -2263,7 +2119,6 @@ dependencies = [ name = "vortex-bytebool" version = "0.1.0" dependencies = [ - "arrow-buffer", "num-traits", "vortex-array", "vortex-buffer", @@ -2274,21 +2129,25 @@ dependencies = [ ] [[package]] -name = "vortex-datetime-parts" +name = "vortex-compute" version = "0.1.0" dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "log", + "multiversion", "num-traits", - "prost", - "vortex-array", + "paste", "vortex-buffer", "vortex-dtype", "vortex-error", "vortex-mask", - "vortex-scalar", + "vortex-vector", ] [[package]] -name = "vortex-decimal-byte-parts" +name = "vortex-datetime-parts" version = "0.1.0" dependencies = [ "num-traits", @@ -2302,27 +2161,24 @@ dependencies = [ ] [[package]] -name = "vortex-dict" +name = "vortex-decimal-byte-parts" version = "0.1.0" dependencies = [ - "arrow-array", - "arrow-buffer", "num-traits", "prost", - "rustc-hash", "vortex-array", "vortex-buffer", "vortex-dtype", "vortex-error", "vortex-mask", "vortex-scalar", - "vortex-utils", ] [[package]] name = "vortex-dtype" version = "0.1.0" dependencies = [ + "arrow-buffer", "arrow-schema", "flatbuffers", "half", @@ -2352,29 +2208,6 @@ dependencies = [ "url", ] -[[package]] -name = "vortex-expr" -version = "0.1.0" -dependencies = [ - "arcref", - "async-trait", - "dyn-hash", - "futures", - "itertools", - "parking_lot", - "paste", - "prost", - "termtree", - "vortex-array", - "vortex-buffer", - "vortex-dtype", - "vortex-error", - "vortex-mask", - "vortex-proto", - "vortex-scalar", - "vortex-utils", -] - [[package]] name = "vortex-fastlanes" version = "0.1.0" @@ -2387,13 +2220,17 @@ dependencies = [ "log", "num-traits", "prost", + "static_assertions", "vortex-array", "vortex-buffer", + "vortex-compute", "vortex-dtype", "vortex-error", "vortex-mask", "vortex-scalar", + "vortex-session", "vortex-utils", + "vortex-vector", ] [[package]] @@ -2417,6 +2254,7 @@ dependencies = [ "vortex-error", "vortex-mask", "vortex-scalar", + "vortex-vector", ] [[package]] @@ -2424,12 +2262,13 @@ name = "vortex-io" version = "0.1.0" dependencies = [ "async-compat", + "async-fs", "async-stream", "async-trait", "bytes", "cfg-if", "futures", - "getrandom 0.3.3", + "getrandom 0.3.4", "handle", "kanal", "log", @@ -2442,6 +2281,7 @@ dependencies = [ "vortex-buffer", "vortex-error", "vortex-metrics", + "vortex-session", "wasm-bindgen-futures", ] @@ -2471,7 +2311,6 @@ dependencies = [ "async-trait", "flatbuffers", "futures", - "getrandom 0.3.3", "itertools", "kanal", "log", @@ -2484,16 +2323,14 @@ dependencies = [ "pin-project-lite", "prost", "rustc-hash", - "tracing", + "termtree", "uuid", "vortex-array", "vortex-btrblocks", "vortex-buffer", "vortex-decimal-byte-parts", - "vortex-dict", "vortex-dtype", "vortex-error", - "vortex-expr", "vortex-flatbuffers", "vortex-io", "vortex-mask", @@ -2501,6 +2338,7 @@ dependencies = [ "vortex-pco", "vortex-scalar", "vortex-sequence", + "vortex-session", "vortex-utils", ] @@ -2508,8 +2346,8 @@ dependencies = [ name = "vortex-mask" version = "0.1.0" dependencies = [ - "arrow-buffer", "itertools", + "vortex-buffer", "vortex-error", ] @@ -2517,8 +2355,9 @@ dependencies = [ name = "vortex-metrics" version = "0.1.0" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", "parking_lot", + "vortex-session", "witchcraft-metrics", ] @@ -2526,14 +2365,17 @@ dependencies = [ name = "vortex-pco" version = "0.1.0" dependencies = [ + "itertools", "pco", "prost", "vortex-array", "vortex-buffer", + "vortex-compute", "vortex-dtype", "vortex-error", "vortex-mask", "vortex-scalar", + "vortex-vector", ] [[package]] @@ -2566,7 +2408,6 @@ name = "vortex-scalar" version = "0.1.0" dependencies = [ "arrow-array", - "arrow-buffer", "bytes", "itertools", "num-traits", @@ -2575,8 +2416,10 @@ dependencies = [ "vortex-buffer", "vortex-dtype", "vortex-error", + "vortex-mask", "vortex-proto", "vortex-utils", + "vortex-vector", ] [[package]] @@ -2586,8 +2429,6 @@ dependencies = [ "arrow-array", "arrow-schema", "bit-vec", - "crossbeam-deque", - "crossbeam-queue", "futures", "itertools", "log", @@ -2597,11 +2438,11 @@ dependencies = [ "vortex-buffer", "vortex-dtype", "vortex-error", - "vortex-expr", "vortex-io", "vortex-layout", "vortex-mask", "vortex-metrics", + "vortex-session", ] [[package]] @@ -2618,6 +2459,16 @@ dependencies = [ "vortex-mask", "vortex-proto", "vortex-scalar", + "vortex-vector", +] + +[[package]] +name = "vortex-session" +version = "0.1.0" +dependencies = [ + "dashmap", + "vortex-error", + "vortex-utils", ] [[package]] @@ -2633,6 +2484,7 @@ dependencies = [ "vortex-error", "vortex-mask", "vortex-scalar", + "vortex-vector", ] [[package]] @@ -2640,13 +2492,26 @@ name = "vortex-utils" version = "0.1.0" dependencies = [ "dashmap", - "hashbrown 0.16.0", + "hashbrown 0.16.1", +] + +[[package]] +name = "vortex-vector" +version = "0.1.0" +dependencies = [ + "paste", + "static_assertions", + "vortex-buffer", + "vortex-dtype", + "vortex-error", + "vortex-mask", ] [[package]] name = "vortex-zigzag" version = "0.1.0" dependencies = [ + "itertools", "vortex-array", "vortex-buffer", "vortex-dtype", @@ -2662,15 +2527,6 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" -[[package]] -name = "wasi" -version = "0.14.7+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" -dependencies = [ - "wasip2", -] - [[package]] name = "wasip2" version = "1.0.1+wasi-0.2.4" @@ -2682,9 +2538,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.104" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" +checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" dependencies = [ "cfg-if", "once_cell", @@ -2693,25 +2549,11 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn 2.0.106", - "wasm-bindgen-shared", -] - [[package]] name = "wasm-bindgen-futures" -version = "0.4.54" +version = "0.4.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e038d41e478cc73bae0ff9b36c60cff1c98b8f38f8d7e8061e79ee63608ac5c" +checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" dependencies = [ "cfg-if", "js-sys", @@ -2722,9 +2564,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.104" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" +checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2732,22 +2574,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.104" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" +checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" dependencies = [ + "bumpalo", "proc-macro2", "quote", - "syn 2.0.106", - "wasm-bindgen-backend", + "syn 2.0.111", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.104" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" +checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" dependencies = [ "unicode-ident", ] @@ -2761,9 +2603,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.81" +version = "0.3.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9367c417a924a74cae129e6a2ae3b47fabb1f8995595ab474029da749a8be120" +checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" dependencies = [ "js-sys", "wasm-bindgen", @@ -2790,7 +2632,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -2801,7 +2643,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -2828,15 +2670,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -2846,70 +2679,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - [[package]] name = "wit-bindgen" version = "0.46.0" @@ -2931,9 +2700,9 @@ dependencies = [ [[package]] name = "writeable" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" [[package]] name = "wyz" @@ -2946,11 +2715,10 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" dependencies = [ - "serde", "stable_deref_trait", "yoke-derive", "zerofrom", @@ -2958,34 +2726,34 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.27" +version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.27" +version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] @@ -3005,15 +2773,15 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", "synstructure", ] [[package]] name = "zerotrie" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" dependencies = [ "displaydoc", "yoke", @@ -3022,9 +2790,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.4" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" dependencies = [ "yoke", "zerofrom", @@ -3033,13 +2801,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.111", ] [[package]] From 08037699ccb526d963df64fea9f18100c2f52853 Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Fri, 12 Dec 2025 12:23:39 -0500 Subject: [PATCH 3/8] port over cast Signed-off-by: Andrew Duffy --- .../src/arrays/struct_/compute/cast.rs | 37 ++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/vortex-array/src/arrays/struct_/compute/cast.rs b/vortex-array/src/arrays/struct_/compute/cast.rs index 0b36668d902..57fddd6cd9a 100644 --- a/vortex-array/src/arrays/struct_/compute/cast.rs +++ b/vortex-array/src/arrays/struct_/compute/cast.rs @@ -5,10 +5,12 @@ use itertools::Itertools; use vortex_dtype::DType; use vortex_error::VortexExpect; use vortex_error::VortexResult; -use vortex_error::vortex_bail; +use vortex_error::vortex_ensure; +use vortex_scalar::Scalar; use crate::ArrayRef; use crate::IntoArray; +use crate::arrays::ConstantArray; use crate::arrays::StructArray; use crate::arrays::StructVTable; use crate::compute::CastKernel; @@ -28,8 +30,30 @@ impl CastKernel for StructVTable { .as_struct_fields_opt() .vortex_expect("struct array must have struct dtype"); - if target_sdtype.names() != source_sdtype.names() { - vortex_bail!("cannot cast {} to {}", array.dtype(), dtype); + // Re-order, handle fields by value instead. + let mut cast_fields = vec![]; + for (target_name, target_type) in + target_sdtype.names().iter().zip_eq(target_sdtype.fields()) + { + match source_sdtype.find(target_name) { + None => { + // No source field with this name => evolve the schema compatibly. + // If the field is nullable, we add a new ConstantArray field with the type. + vortex_ensure!( + target_type.is_nullable(), + "CAST for struct only supports added nullable fields" + ); + + cast_fields.push( + ConstantArray::new(Scalar::null(target_type), array.len).into_array(), + ); + } + Some(src_field_idx) => { + // Field exists in source field. Cast it to the target type. + let cast_field = cast(array.fields()[src_field_idx].as_ref(), &target_type)?; + cast_fields.push(cast_field); + } + } } let validity = array @@ -39,12 +63,7 @@ impl CastKernel for StructVTable { StructArray::try_new( target_sdtype.names().clone(), - array - .fields() - .iter() - .zip_eq(target_sdtype.fields()) - .map(|(field, dtype)| cast(field, &dtype)) - .collect::, _>>()?, + cast_fields, array.len(), validity, ) From bdc2db2104b12a89b3caa3324ff45263ee8895d5 Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Fri, 12 Dec 2025 12:55:42 -0500 Subject: [PATCH 4/8] cleanup opener Signed-off-by: Andrew Duffy --- vortex-datafusion/src/convert/exprs.rs | 6 +- vortex-datafusion/src/persistent/opener.rs | 310 ++++++++++----------- 2 files changed, 155 insertions(+), 161 deletions(-) diff --git a/vortex-datafusion/src/convert/exprs.rs b/vortex-datafusion/src/convert/exprs.rs index cdd194482c8..f95ea6d8eb2 100644 --- a/vortex-datafusion/src/convert/exprs.rs +++ b/vortex-datafusion/src/convert/exprs.rs @@ -40,7 +40,7 @@ use crate::convert::TryFromDataFusion; /// Tries to convert the expressions into a vortex conjunction. Will return Ok(None) iff the input conjunction is empty. pub(crate) fn make_vortex_predicate( - predicate: &[&Arc], + predicate: &[Arc], ) -> VortexResult> { let exprs = predicate .iter() @@ -391,7 +391,7 @@ mod tests { #[test] fn test_make_vortex_predicate_single() { let col_expr = Arc::new(df_expr::Column::new("test", 0)) as Arc; - let result = make_vortex_predicate(&[&col_expr]).unwrap(); + let result = make_vortex_predicate(&[col_expr]).unwrap(); assert!(result.is_some()); } @@ -399,7 +399,7 @@ mod tests { fn test_make_vortex_predicate_multiple() { let col1 = Arc::new(df_expr::Column::new("col1", 0)) as Arc; let col2 = Arc::new(df_expr::Column::new("col2", 1)) as Arc; - let result = make_vortex_predicate(&[&col1, &col2]).unwrap(); + let result = make_vortex_predicate(&[col1, col2]).unwrap(); assert!(result.is_some()); // Result should be an AND expression combining the two columns } diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index 84c4e84128b..0566cc996a2 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -6,9 +6,6 @@ use std::sync::Arc; use std::sync::Weak; use arrow_schema::ArrowError; -use arrow_schema::DataType; -use arrow_schema::Field; -use arrow_schema::SchemaRef; use datafusion_common::DataFusionError; use datafusion_common::Result as DFResult; use datafusion_common::arrow::array::RecordBatch; @@ -35,6 +32,8 @@ use tracing::Instrument; use vortex::array::ArrayRef; use vortex::dtype::FieldName; use vortex::error::VortexError; +use vortex::error::VortexResult; +use vortex::error::vortex_err; use vortex::expr::root; use vortex::expr::select; use vortex::layout::LayoutReader; @@ -52,7 +51,8 @@ use crate::convert::exprs::make_vortex_predicate; pub(crate) struct VortexOpener { pub session: VortexSession, pub object_store: Arc, - /// Projection by index of the file's columns + /// Optional table schema projection. The indices are w.r.t. the `table_schema`, which is + /// all fields in the final scan result not including the partition columns. pub projection: Option>, /// Filter expression optimized for pushdown into Vortex scan operations. /// This may be a subset of file_pruning_predicate containing only expressions @@ -63,97 +63,26 @@ pub(crate) struct VortexOpener { pub file_pruning_predicate: Option, pub expr_adapter_factory: Option>, pub schema_adapter_factory: Arc, + /// This is the table's schema without partition columns. It may contain fields which do + /// not exist in the file, and are supplied by the `schema_adapter_factory`. pub table_schema: TableSchema, + /// Caching Vortex file opener pub file_cache: VortexFileCache, + /// A hint for the desired row count of record batches returned from the scan. pub batch_size: usize, + /// If provided, the scan will not return more than this many rows. pub limit: Option, + /// A metrics object for tracking performance of the scan. pub metrics: VortexMetrics, + /// A shared cache of file readers. + /// + /// To save on the overhead of reparsing FlatBuffers and rebuilding the layout tree, we cache + /// a file reader the first time we read a file. pub layout_readers: Arc>>, /// Whether the query has output ordering specified pub has_output_ordering: bool, } -/// Merges the data types of two fields, preferring the logical type from the -/// table field. -fn merge_field_types(physical_field: &Field, table_field: &Field) -> DataType { - match (physical_field.data_type(), table_field.data_type()) { - (DataType::Struct(phys_fields), DataType::Struct(table_fields)) => { - let merged_fields = merge_fields(phys_fields, table_fields); - DataType::Struct(merged_fields.into()) - } - (DataType::List(phys_field), DataType::List(table_field)) => { - DataType::List(Arc::new(Field::new( - phys_field.name(), - merge_field_types(phys_field, table_field), - phys_field.is_nullable(), - ))) - } - (DataType::LargeList(phys_field), DataType::LargeList(table_field)) => { - DataType::LargeList(Arc::new(Field::new( - phys_field.name(), - merge_field_types(phys_field, table_field), - phys_field.is_nullable(), - ))) - } - _ => table_field.data_type().clone(), - } -} - -/// Merges two field collections, using logical types from table_fields where available. -/// Falls back to physical field types when no matching table field is found. -fn merge_fields( - physical_fields: &arrow_schema::Fields, - table_fields: &arrow_schema::Fields, -) -> Vec { - physical_fields - .iter() - .map(|phys_field| { - table_fields - .iter() - .find(|f| f.name() == phys_field.name()) - .map(|table_field| { - Field::new( - phys_field.name(), - merge_field_types(phys_field, table_field), - phys_field.is_nullable(), - ) - }) - .unwrap_or_else(|| (**phys_field).clone()) - }) - .collect() -} - -/// Computes a logical file schema from the physical file schema and the table -/// schema. -/// -/// For each field in the physical file schema, looks up the corresponding field -/// in the table schema and uses its logical type. -fn compute_logical_file_schema( - physical_file_schema: &SchemaRef, - table_schema: &SchemaRef, -) -> SchemaRef { - let logical_fields: Vec = physical_file_schema - .fields() - .iter() - .map(|physical_field| { - table_schema - .fields() - .find(physical_field.name()) - .map(|(_, table_field)| { - Field::new( - physical_field.name(), - merge_field_types(physical_field, table_field), - physical_field.is_nullable(), - ) - .with_metadata(physical_field.metadata().clone()) - }) - .unwrap_or_else(|| (**physical_field).clone()) - }) - .collect(); - - Arc::new(arrow_schema::Schema::new(logical_fields)) -} - impl FileOpener for VortexOpener { fn open(&self, file: PartitionedFile) -> DFResult { let session = self.session.clone(); @@ -175,12 +104,17 @@ impl FileOpener for VortexOpener { Some(indices) => Arc::new(table_schema.file_schema().project(indices)?), }; - let mut predicate_file_schema = table_schema.file_schema().clone(); + let _file_name = file.object_meta.location.to_string(); + let _schema = projected_schema.to_string(); let schema_adapter = self .schema_adapter_factory .create(projected_schema, table_schema.table_schema().clone()); + // Update partition column access in the filter to use literals instead + let partition_fields = self.table_schema.table_partition_cols().clone(); + let table_schema = self.table_schema.clone(); + Ok(async move { // Create FilePruner when we have a predicate and either dynamic expressions // or file statistics available. The pruner can eliminate files without @@ -188,23 +122,21 @@ impl FileOpener for VortexOpener { // - Partition column values (e.g., date=2024-01-01) // - File-level statistics (min/max values per column) let mut file_pruner = file_pruning_predicate - .map(|predicate| { + .filter(|p| { // Only create pruner if we have dynamic expressions or file statistics // to work with. Static predicates without stats won't benefit from pruning. - Ok::<_, DataFusionError>( - (is_dynamic_physical_expr(&predicate) | file.has_statistics()).then_some( - FilePruner::new( - predicate.clone(), - table_schema.file_schema(), - table_schema.table_partition_cols().clone(), - file.clone(), - Count::default(), - )?, - ), + is_dynamic_physical_expr(p) || file.has_statistics() + }) + .map(|predicate| { + FilePruner::new( + predicate.clone(), + table_schema.file_schema(), + partition_fields, + file.clone(), + Count::default(), ) }) - .transpose()? - .flatten(); + .transpose()?; // Check if this file should be pruned based on statistics/partition values. // Returns empty stream if file can be skipped entirely. @@ -225,26 +157,26 @@ impl FileOpener for VortexOpener { DataFusionError::Execution(format!("Failed to convert file schema to arrow: {e}")) })?); - // Compute the logical file schema by merging physical file types with logical table types. - // This schema has the same field names as logical_schema, but with physical types from the file. - let logical_file_schema = - compute_logical_file_schema(&physical_file_schema, table_schema.file_schema()); - if let Some(expr_adapter_factory) = expr_adapter_factory { - let partition_values = table_schema - .table_partition_cols() - .iter() - .cloned() - .zip(file.partition_values) - .collect::>(); + // Replace column access for partition columns with literals // The adapter rewrites the expression to the local file schema, allowing // for schema evolution and divergence between the table's schema and individual files. filter = filter .map(|filter| { let expr = expr_adapter_factory - .create(logical_file_schema.clone(), physical_file_schema.clone()) - .with_partition_values(partition_values) + .create( + Arc::clone(table_schema.file_schema()), + Arc::clone(&physical_file_schema), + ) + .with_partition_values( + table_schema + .table_partition_cols() + .iter() + .cloned() + .zip(file.partition_values.iter().cloned()) + .collect(), + ) .rewrite(filter)?; // Expression might now reference columns that don't exist in the file, so we can give it @@ -252,25 +184,20 @@ impl FileOpener for VortexOpener { PhysicalExprSimplifier::new(&physical_file_schema).simplify(expr) }) .transpose()?; - - predicate_file_schema = physical_file_schema; } // Use the pre-created schema adapter to map logical_file_schema to projected_schema. // Since logical_file_schema has the same field names as logical_schema (which the adapter // was created with), this works correctly and gives us the projection indices. let (schema_mapping, adapted_projections) = - schema_adapter.map_schema(&logical_file_schema)?; + schema_adapter.map_schema(&physical_file_schema)?; // Build the Vortex projection expression using field names from logical_file_schema - let fields = adapted_projections - .iter() - .map(|&idx| { - let field = logical_file_schema.field(idx); - FieldName::from(field.name().as_str()) - }) - .collect::>(); - let projection_expr = select(fields, root()); + let field_names: Vec = adapted_projections + .into_iter() + .map(|index| FieldName::from(physical_file_schema.field(index).name().as_str())) + .collect(); + let projection_expr = select(field_names, root()); // We share our layout readers with others partitions in the scan, so we can only need to read each layout in each file once. let layout_reader = match layout_reader.entry(file.object_meta.location.clone()) { @@ -312,12 +239,32 @@ impl FileOpener for VortexOpener { let filter = filter .and_then(|f| { - let exprs = split_conjunction(&f) - .into_iter() - .filter(|expr| can_be_pushed_down(expr, &predicate_file_schema)) - .collect::>(); + // Verify that all filters we've accepted from DataFusion get pushed down. + // This will only fail if the user has not configured a suitable + // PhysicalExprAdapterFactory on the file source to handle rewriting the + // expression to handle missing/reordered columns in the Vortex file. + + let (pushed, unpushed): (Vec, Vec) = + split_conjunction(&f) + .into_iter() + .cloned() + .partition(|expr| can_be_pushed_down(expr, &physical_file_schema)); + + if !unpushed.is_empty() { + return Some(VortexResult::Err(vortex_err!( + r#"VortexSource accepted but failed to push {} filters. + This should never happen if you have a properly configured + PhysicalExprAdapterFactory configured on the source. + + Failed filters: + + {unpushed:#?} + "#, + unpushed.len() + ))); + } - make_vortex_predicate(&exprs).transpose() + make_vortex_predicate(&pushed).transpose() }) .transpose() .map_err(|e| DataFusionError::External(e.into()))?; @@ -407,6 +354,7 @@ fn byte_range_to_row_range(byte_range: Range, row_count: u64, total_size: u mod tests { use std::sync::LazyLock; + use arrow_schema::Field; use arrow_schema::Fields; use datafusion::arrow::array::RecordBatch; use datafusion::arrow::array::StringArray; @@ -497,16 +445,36 @@ mod tests { Ok(summary.size()) } - #[rstest] - #[case(Some(Arc::new(DefaultPhysicalExprAdapterFactory) as _), (1, 3), (0, 0))] - // If we don't have a physical expr adapter, we just drop filters on partition values - #[case(None, (1, 3), (1, 3))] + fn make_opener( + object_store: Arc, + table_schema: TableSchema, + filter: Option, + expr_adapter_factory: Option>, + ) -> VortexOpener { + VortexOpener { + session: SESSION.clone(), + object_store, + projection: Some([0].into()), + filter, + file_pruning_predicate: None, + // no adapter + expr_adapter_factory, + schema_adapter_factory: Arc::new(DefaultSchemaAdapterFactory), + file_cache: VortexFileCache::new(1, 1, SESSION.clone()), + table_schema, + batch_size: 100, + limit: None, + metrics: Default::default(), + layout_readers: Default::default(), + has_output_ordering: false, + } + } + #[tokio::test] - async fn test_adapter_optimization_partition_column( - #[case] expr_adapter_factory: Option>, - #[case] expected_result1: (usize, usize), - #[case] expected_result2: (usize, usize), - ) -> anyhow::Result<()> { + async fn test_open_with_adapter() -> anyhow::Result<()> { + let expr_adapter_factory: Arc = + Arc::new(DefaultPhysicalExprAdapterFactory); + let object_store = Arc::new(InMemory::new()) as Arc; let file_path = "part=1/file.vortex"; let batch = record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3)])).unwrap(); @@ -522,47 +490,73 @@ mod tests { vec![Arc::new(Field::new("part", DataType::Int32, false))], ); - let make_opener = |filter| VortexOpener { - session: SESSION.clone(), - object_store: object_store.clone(), - projection: Some([0].into()), - filter: Some(filter), - file_pruning_predicate: None, - expr_adapter_factory: expr_adapter_factory.clone(), - schema_adapter_factory: Arc::new(DefaultSchemaAdapterFactory), - file_cache: VortexFileCache::new(1, 1, SESSION.clone()), - table_schema: table_schema.clone(), - batch_size: 100, - limit: None, - metrics: Default::default(), - layout_readers: Default::default(), - has_output_ordering: false, - }; - // filter matches partition value let filter = col("part").eq(lit(1)); let filter = logical2physical(&filter, table_schema.table_schema()); - let opener = make_opener(filter); + let opener = make_opener( + object_store.clone(), + table_schema.clone(), + Some(filter), + Some(expr_adapter_factory.clone()), + ); let stream = opener.open(file.clone()).unwrap().await.unwrap(); let data = stream.try_collect::>().await?; let num_batches = data.len(); let num_rows = data.iter().map(|rb| rb.num_rows()).sum::(); - assert_eq!((num_batches, num_rows), expected_result1); + assert_eq!((num_batches, num_rows), (1, 3)); // filter doesn't matches partition value let filter = col("part").eq(lit(2)); let filter = logical2physical(&filter, table_schema.table_schema()); - let opener = make_opener(filter); + let opener = make_opener( + object_store.clone(), + table_schema.clone(), + Some(filter), + Some(expr_adapter_factory), + ); let stream = opener.open(file.clone()).unwrap().await.unwrap(); let data = stream.try_collect::>().await?; let num_batches = data.len(); let num_rows = data.iter().map(|rb| rb.num_rows()).sum::(); - assert_eq!((num_batches, num_rows), expected_result2); + assert_eq!((num_batches, num_rows), (0, 0)); + + Ok(()) + } + #[tokio::test] + async fn test_open_without_adapter() -> anyhow::Result<()> { + let object_store = Arc::new(InMemory::new()) as Arc; + let file_path = "part=1/file.vortex"; + let batch = record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3)])).unwrap(); + let data_size = + write_arrow_to_vortex(object_store.clone(), file_path, batch.clone()).await?; + + let file_schema = batch.schema(); + let mut file = PartitionedFile::new(file_path.to_string(), data_size); + file.partition_values = vec![ScalarValue::Int32(Some(1))]; + + let table_schema = TableSchema::new( + file_schema.clone(), + vec![Arc::new(Field::new("part", DataType::Int32, false))], + ); + + // filter matches partition value + let filter = col("part").eq(lit(1)); + let filter = logical2physical(&filter, table_schema.table_schema()); + + let opener = make_opener( + object_store.clone(), + table_schema.clone(), + Some(filter), + None, + ); + + let result = opener.open(file.clone()).unwrap().await; + assert!(result.is_err()); Ok(()) } From 9f58560f9e896b1d4d433afa56fd2f959ea09a86 Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Fri, 12 Dec 2025 13:02:57 -0500 Subject: [PATCH 5/8] add more Signed-off-by: Andrew Duffy --- Cargo.lock | 1 + vortex-datafusion/Cargo.toml | 1 + vortex-datafusion/src/convert/exprs.rs | 68 +--- vortex-datafusion/src/persistent/opener.rs | 30 +- vortex-datafusion/tests/schema_evolution.rs | 424 ++++++++++++++++++++ 5 files changed, 456 insertions(+), 68 deletions(-) create mode 100644 vortex-datafusion/tests/schema_evolution.rs diff --git a/Cargo.lock b/Cargo.lock index 64f2af3c92d..4f88017e076 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9377,6 +9377,7 @@ dependencies = [ "tokio", "tokio-stream", "tracing", + "url", "vortex", "vortex-utils", "walkdir", diff --git a/vortex-datafusion/Cargo.toml b/vortex-datafusion/Cargo.toml index c929b1befc1..9a5c610140a 100644 --- a/vortex-datafusion/Cargo.toml +++ b/vortex-datafusion/Cargo.toml @@ -47,6 +47,7 @@ insta = { workspace = true } rstest = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true, features = ["test-util", "rt-multi-thread", "fs"] } +url = { workspace = true } walkdir = { workspace = true } [lints] diff --git a/vortex-datafusion/src/convert/exprs.rs b/vortex-datafusion/src/convert/exprs.rs index f95ea6d8eb2..c6270a23d49 100644 --- a/vortex-datafusion/src/convert/exprs.rs +++ b/vortex-datafusion/src/convert/exprs.rs @@ -9,7 +9,6 @@ use datafusion_expr::Operator as DFOperator; use datafusion_functions::core::getfield::GetFieldFunc; use datafusion_physical_expr::PhysicalExpr; use datafusion_physical_expr::ScalarFunctionExpr; -use datafusion_physical_expr_common::physical_expr::PhysicalExprRef; use datafusion_physical_expr_common::physical_expr::is_dynamic_physical_expr; use datafusion_physical_plan::expressions as df_expr; use itertools::Itertools; @@ -89,6 +88,14 @@ impl TryFromDataFusion for Expression { return Ok(cast(child, cast_dtype)); } + if let Some(cast_col_expr) = df.as_any().downcast_ref::() { + let target = cast_col_expr.target_field(); + + let target_dtype = DType::from_arrow((target.data_type(), target.is_nullable().into())); + let child = Expression::try_from_df(cast_col_expr.expr().as_ref())?; + return Ok(cast(child, target_dtype)); + } + if let Some(is_null_expr) = df.as_any().downcast_ref::() { let arg = Expression::try_from_df(is_null_expr.arg().as_ref())?; return Ok(is_null(arg)); @@ -216,7 +223,7 @@ impl TryFromDataFusion for Operator { } } -pub(crate) fn can_be_pushed_down(df_expr: &PhysicalExprRef, schema: &Schema) -> bool { +pub(crate) fn can_be_pushed_down(df_expr: &Arc, schema: &Schema) -> bool { // We currently do not support pushdown of dynamic expressions in DF. // See issue: https://github.com/vortex-data/vortex/issues/4034 if is_dynamic_physical_expr(df_expr) { @@ -235,8 +242,10 @@ pub(crate) fn can_be_pushed_down(df_expr: &PhysicalExprRef, schema: &Schema) -> can_be_pushed_down(like.expr(), schema) && can_be_pushed_down(like.pattern(), schema) } else if let Some(lit) = expr.downcast_ref::() { supported_data_types(&lit.value().data_type()) - } else if let Some(cast) = expr.downcast_ref::() { - supported_data_types(cast.cast_type()) && can_be_pushed_down(cast.expr(), schema) + } else if expr.downcast_ref::().is_some() + || expr.downcast_ref::().is_some() + { + true } else if let Some(is_null) = expr.downcast_ref::() { can_be_pushed_down(is_null.arg(), schema) } else if let Some(is_not_null) = expr.downcast_ref::() { @@ -245,7 +254,7 @@ pub(crate) fn can_be_pushed_down(df_expr: &PhysicalExprRef, schema: &Schema) -> can_be_pushed_down(in_list.expr(), schema) && in_list.list().iter().all(|e| can_be_pushed_down(e, schema)) } else if let Some(scalar_fn) = expr.downcast_ref::() { - can_scalar_fn_be_pushed_down(scalar_fn, schema) + can_scalar_fn_be_pushed_down(scalar_fn) } else { tracing::debug!(%df_expr, "DataFusion expression can't be pushed down"); false @@ -293,50 +302,8 @@ fn supported_data_types(dt: &DataType) -> bool { } /// Checks if a GetField scalar function can be pushed down. -fn can_scalar_fn_be_pushed_down(scalar_fn: &ScalarFunctionExpr, schema: &Schema) -> bool { - let Some(get_field_fn) = ScalarFunctionExpr::try_downcast_func::(scalar_fn) - else { - // Only get_field pushdown is supported. - return false; - }; - - let args = get_field_fn.args(); - if args.len() != 2 { - tracing::debug!( - "Expected 2 arguments for GetField, not pushing down {} arguments", - args.len() - ); - return false; - } - let source_expr = &args[0]; - let field_name_expr = &args[1]; - let Some(field_name) = field_name_expr - .as_any() - .downcast_ref::() - .and_then(|lit| lit.value().try_as_str().flatten()) - else { - return false; - }; - - let Ok(source_dt) = source_expr.data_type(schema) else { - tracing::debug!( - field_name = field_name, - schema = ?schema, - source_expr = ?source_expr, - "Failed to get source type for GetField, not pushing down" - ); - return false; - }; - let DataType::Struct(fields) = source_dt else { - tracing::debug!( - field_name = field_name, - schema = ?schema, - source_expr = ?source_expr, - "Failed to get source type as struct for GetField, not pushing down" - ); - return false; - }; - fields.find(field_name).is_some() +fn can_scalar_fn_be_pushed_down(scalar_fn: &ScalarFunctionExpr) -> bool { + ScalarFunctionExpr::try_downcast_func::(scalar_fn).is_some() } #[cfg(test)] @@ -532,7 +499,8 @@ mod tests { DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), false )] - #[case::struct_type(DataType::Struct(vec![Field::new("field", DataType::Int32, true)].into()), false)] + #[case::struct_type(DataType::Struct(vec![Field::new("field", DataType::Int32, true)].into() + ), false)] // Dictionary types - should be supported if value type is supported #[case::dict_utf8( DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)), diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index 0566cc996a2..21814882f69 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -767,33 +767,27 @@ mod tests { true, )]))); - let opener = VortexOpener { - session: SESSION.clone(), - object_store: object_store.clone(), - projection: None, - filter: Some(logical2physical( + let opener = make_opener( + object_store.clone(), + table_schema.clone(), + // expression references my_struct column which has different fields in each + // field. + Some(logical2physical( &col("my_struct").is_not_null(), table_schema.table_schema(), )), - file_pruning_predicate: None, - expr_adapter_factory: Some(Arc::new(DefaultPhysicalExprAdapterFactory) as _), - schema_adapter_factory: Arc::new(DefaultSchemaAdapterFactory), - file_cache: VortexFileCache::new(1, 1, SESSION.clone()), - table_schema: table_schema.clone(), - batch_size: 100, - limit: None, - metrics: Default::default(), - layout_readers: Default::default(), - has_output_ordering: false, - }; + Some(Arc::new(DefaultPhysicalExprAdapterFactory) as _), + ); // The opener should be able to open the file with a filter on the // struct column. let data = opener .open(PartitionedFile::new(file_path.to_string(), data_size))? - .await? + .await + .unwrap() .try_collect::>() - .await?; + .await + .unwrap(); assert_eq!(data.len(), 1); assert_eq!(data[0].num_rows(), 3); diff --git a/vortex-datafusion/tests/schema_evolution.rs b/vortex-datafusion/tests/schema_evolution.rs new file mode 100644 index 00000000000..90d83eef978 --- /dev/null +++ b/vortex-datafusion/tests/schema_evolution.rs @@ -0,0 +1,424 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#![allow( + clippy::unwrap_in_result, + clippy::unwrap_used, + clippy::tests_outside_test_module +)] + +//! Test that checks we can evolve schemas in a compatible way across files. + +use std::sync::Arc; +use std::sync::LazyLock; + +use arrow_schema::DataType; +use arrow_schema::Field; +use arrow_schema::Schema; +use arrow_schema::SchemaRef; +use datafusion::arrow::array::Array; +use datafusion::arrow::array::ArrayRef as ArrowArrayRef; +use datafusion::arrow::array::Int32Array; +use datafusion::arrow::array::RecordBatch; +use datafusion::arrow::array::StringViewArray; +use datafusion::arrow::array::StructArray; +use datafusion::arrow::compute::concat_batches; +use datafusion::datasource::listing::ListingOptions; +use datafusion::datasource::listing::ListingTable; +use datafusion::datasource::listing::ListingTableConfig; +use datafusion::execution::SessionStateBuilder; +use datafusion::execution::context::SessionContext; +use datafusion_common::create_array; +use datafusion_common::record_batch; +use datafusion_datasource::ListingTableUrl; +use datafusion_expr::col; +use datafusion_expr::lit; +use datafusion_functions::expr_fn::get_field; +use object_store::ObjectStore; +use object_store::memory::InMemory; +use object_store::path::Path; +use url::Url; +use vortex::VortexSessionDefault; +use vortex::array::ArrayRef; +use vortex::array::arrow::FromArrowArray; +use vortex::file::WriteOptionsSessionExt; +use vortex::io::ObjectStoreWriter; +use vortex::io::VortexWrite; +use vortex::session::VortexSession; +use vortex_datafusion::VortexFormat; +use vortex_datafusion::VortexFormatFactory; + +static SESSION: LazyLock = LazyLock::new(VortexSession::default); + +fn register_vortex_format_factory( + factory: VortexFormatFactory, + session_state_builder: &mut SessionStateBuilder, +) { + if let Some(table_factories) = session_state_builder.table_factories() { + table_factories.insert( + datafusion::common::GetExt::get_ext(&factory).to_uppercase(), // Has to be uppercase + Arc::new(datafusion::datasource::provider::DefaultTableFactory::new()), + ); + } + + if let Some(file_formats) = session_state_builder.file_formats() { + file_formats.push(Arc::new(factory)); + } +} + +fn make_session_ctx() -> (SessionContext, Arc) { + let factory: VortexFormatFactory = VortexFormatFactory::new(); + let mut session_state_builder = SessionStateBuilder::new().with_default_features(); + register_vortex_format_factory(factory, &mut session_state_builder); + let ctx = SessionContext::new_with_state(session_state_builder.build()); + let store = Arc::new(InMemory::new()); + ctx.register_object_store(&Url::parse("s3://in-memory/").unwrap(), store.clone()); + + (ctx, store) +} + +async fn write_file(store: &Arc, path: &str, records: &RecordBatch) { + let array = ArrayRef::from_arrow(records, false); + let path = Path::from_url_path(path).unwrap(); + let mut write = ObjectStoreWriter::new(store.clone(), &path).await.unwrap(); + SESSION + .write_options() + .write(&mut write, array.to_array_stream()) + .await + .unwrap(); + write.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_filter_with_schema_evolution() { + let (ctx, store) = make_session_ctx(); + + // file1 only contains field "a" + write_file( + &store, + "files/file1.vortex", + &record_batch!(("a", Utf8, vec![Some("one"), Some("two"), Some("three")])).unwrap(), + ) + .await; + + // file2 only contains field "b" + write_file( + &store, + "files/file2.vortex", + &record_batch!(("b", Utf8, vec![Some("four"), Some("five"), Some("six")])).unwrap(), + ) + .await; + + // Read the table back as Vortex + let table_url = ListingTableUrl::parse("s3://in-memory/files").unwrap(); + let list_opts = ListingOptions::new(Arc::new(VortexFormat::new(SESSION.clone()))) + .with_session_config_options(ctx.state().config()) + .with_file_extension("vortex"); + + let table = ListingTable::try_new( + ListingTableConfig::new(table_url) + .with_listing_options(list_opts) + .infer_schema(&ctx.state()) + .await + .unwrap(), + ) + .unwrap(); + + let table = Arc::new(table); + + let df = ctx.read_table(table).unwrap(); + + let table_schema = Arc::new(df.schema().as_arrow().clone()); + + // Table schema contains both fields + assert_eq!( + table_schema.as_ref(), + &Schema::new(vec![ + Field::new("a", DataType::Utf8View, true), + Field::new("b", DataType::Utf8View, true), + ]) + ); + + // Filter the result to only ones with a column, i.e. only file1 + let result = df + .filter(col("a").is_not_null()) + .unwrap() + .collect() + .await + .unwrap(); + let table = concat_batches(&table_schema, result.iter()).unwrap(); + + // We read back the full table, with nulls filled in for missing fields + assert_eq!( + table, + record_batch( + &table_schema, + vec![ + // a + Arc::new(StringViewArray::from(vec![ + Some("one"), + Some("two"), + Some("three"), + ])) as ArrowArrayRef, + // b + Arc::new(StringViewArray::from(vec![ + Option::<&str>::None, + None, + None + ])) as ArrowArrayRef, + ] + ) + ); +} + +#[tokio::test] +async fn test_filter_schema_evolution_order() { + let (ctx, store) = make_session_ctx(); + + // file1 only contains field "a" + write_file( + &store, + "files/file1.vortex", + &record_batch!(("a", Int32, vec![Some(1), Some(3), Some(5)])).unwrap(), + ) + .await; + + // file2 containing fields "b" and "a", where "a" needs to be upcast at scan time. + write_file( + &store, + "files/file2.vortex", + &record_batch!( + ("b", Utf8, vec![Some("two"), Some("four"), Some("six")]), + ("a", Int16, vec![Some(2), Some(4), Some(6)]) + ) + .unwrap(), + ) + .await; + + // Read the table back as Vortex + let table_url = ListingTableUrl::parse("s3://in-memory/files").unwrap(); + let list_opts = ListingOptions::new(Arc::new(VortexFormat::new(SESSION.clone()))) + .with_session_config_options(ctx.state().config()) + .with_file_extension("vortex"); + + // We force the table schema, because file1/file2 have different types for the "a" column + let read_schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Utf8View, true), + ])); + + let table = ListingTable::try_new( + ListingTableConfig::new(table_url) + .with_listing_options(list_opts) + .with_schema(read_schema.clone()), + ) + .unwrap(); + + let table = Arc::new(table); + + let df = ctx.read_table(table.clone()).unwrap(); + + let table_schema = Arc::new(df.schema().as_arrow().clone()); + + // Table schema contains both fields + assert_eq!( + table_schema.as_ref(), + &Schema::new(vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Utf8View, true), + ]) + ); + + // Filter referencing the b column, which only appears in file2 + let result = df + .filter(col("b").eq(lit("two"))) + .unwrap() + .collect() + .await + .unwrap(); + let result = concat_batches(&table_schema, result.iter()).unwrap(); + + assert_eq!( + result, + record_batch( + &table_schema, + vec![ + // a + Arc::new(Int32Array::from(vec![Some(2)])) as ArrowArrayRef, + // b + Arc::new(StringViewArray::from(vec![Some("two"),])) as ArrowArrayRef, + ] + ) + ); + + // Filter on the "a" column, which has different types for each file + let result = ctx + .read_table(table) + .unwrap() + .filter(col("a").gt_eq(lit(3i16))) + .unwrap() + .collect() + .await + .unwrap(); + let table = concat_batches(&table_schema, result.iter()).unwrap(); + + // file1, then file2 + assert_eq!( + table, + record_batch( + &table_schema, + vec![ + // a field: present in both files + Arc::new(Int32Array::from(vec![Some(3), Some(5), Some(4), Some(6)])) + as ArrowArrayRef, + // b field: only present in file2, file1 fills with nulls + Arc::new(StringViewArray::from(vec![ + None, + None, + Some("four"), + Some("six") + ])) as ArrowArrayRef, + ] + ) + ); +} + +#[tokio::test] +async fn test_filter_schema_evolution_struct_fields() { + // Test for correct schema evolution behavior in the presence of nested struct fields. + // We use a hypothetical schema of some observability data with "wide records", struct columns + // with nullable payloads that may or may not be present for every file. + + let (ctx, store) = make_session_ctx(); + + fn make_metrics( + hostname: &str, + uptime: Vec, + instance: Option>>, + ) -> RecordBatch { + let values_array: ArrowArrayRef = create_array!(Int64, uptime); + let payload_array = if let Some(tags) = instance { + let tags_array: ArrowArrayRef = create_array!(Utf8, tags); + Arc::new(StructArray::new( + vec![ + Field::new("uptime", DataType::Int64, true), + Field::new("instance", DataType::Utf8, true), + ] + .into(), + vec![values_array, tags_array], + None, + )) + } else { + Arc::new(StructArray::new( + vec![Field::new("uptime", DataType::Int64, true)].into(), + vec![values_array], + None, + )) + }; + + let len = payload_array.len(); + let hostname_array = create_array!(Utf8, vec![Some(hostname); len]); + + let payload_type = payload_array.data_type().clone(); + let hostname_type = hostname_array.data_type().clone(); + + RecordBatch::from(StructArray::new( + vec![ + Field::new("hostname", hostname_type, true), + Field::new("payload", payload_type, true), + ] + .into(), + vec![hostname_array, payload_array], + None, + )) + } + + let host01 = make_metrics("host01.local", vec![1, 2, 3, 4], None); + let host02 = make_metrics( + "host02.local", + vec![10, 20, 30, 40], + // host02 has new logging code which adds the new "instance" nested field in its payload + Some(vec![Some("c6i"), Some("c6i"), Some("m5"), Some("r5")]), + ); + + // Write metrics files to storage + write_file(&store, "files/host01.vortex", &host01).await; + write_file(&store, "files/host02.vortex", &host02).await; + + // Read the table back as Vortex + let table_url = ListingTableUrl::parse("s3://in-memory/files").unwrap(); + let list_opts = ListingOptions::new(Arc::new(VortexFormat::new(SESSION.clone()))) + .with_session_config_options(ctx.state().config()) + .with_file_extension("vortex"); + + // We force the table schema to be the one inclusive of the new instance field. + let read_schema = host02.schema(); + + let table = ListingTable::try_new( + ListingTableConfig::new(table_url) + .with_listing_options(list_opts) + .with_schema(read_schema.clone()), + ) + .unwrap(); + + let table = Arc::new(table); + + let df = ctx.read_table(table.clone()).unwrap(); + + let table_schema = Arc::new(df.schema().as_arrow().clone()); + + // Table schema contains both fields + assert_eq!(table_schema.as_ref(), read_schema.as_ref(),); + + // Scan all the records, NULLs are filled in for nested optional fields. + let full_scan = df.collect().await.unwrap(); + let full_scan = concat_batches(&table_schema, full_scan.iter()).unwrap(); + + let expected = concat_batches( + &table_schema, + &[ + // host01 with extra nulls for the payload.instance field + make_metrics("host01.local", vec![1, 2, 3, 4], Some(vec![None; 4])), + host02, + ], + ) + .unwrap(); + assert_eq!(full_scan, expected); + + // run a filter that touches both the payload.uptime AND the payload.instance nested fields + let df = ctx.read_table(table.clone()).unwrap(); + let filtered_scan = df + .filter( + // payload.instance = 'c6i' OR payload.uptime < 10 + // We need to perform filtering over nested columns which don't exist in every + // file type. + get_field(col("payload"), "instance") + .eq(lit("c6i")) + .or(get_field(col("payload"), "uptime").lt(lit(10))), + ) + .unwrap() + .collect() + .await + .unwrap(); + let filtered_scan = concat_batches(&table_schema, filtered_scan.iter()).unwrap(); + let expected = concat_batches( + &table_schema, + &[ + make_metrics("host01.local", vec![1, 2, 3, 4], Some(vec![None; 4])), + make_metrics( + "host02.local", + vec![10, 20], + Some(vec![Some("c6i"), Some("c6i")]), + ), + ], + ) + .unwrap(); + assert_eq!(filtered_scan, expected); +} + +fn record_batch( + schema: &SchemaRef, + fields: impl IntoIterator, +) -> RecordBatch { + RecordBatch::try_new(schema.clone(), fields.into_iter().collect()).unwrap() +} From 025a6b5b64377869842cf6d17b56f756a52ff024 Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Fri, 12 Dec 2025 13:56:23 -0500 Subject: [PATCH 6/8] almost there... Signed-off-by: Andrew Duffy --- vortex-datafusion/src/persistent/opener.rs | 4 + vortex-datafusion/tests/schema_evolution.rs | 149 +++++++++----------- 2 files changed, 68 insertions(+), 85 deletions(-) diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index 21814882f69..ca50664689d 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -264,6 +264,10 @@ impl FileOpener for VortexOpener { ))); } + for filter in pushed.iter() { + println!("- pushing: {filter}"); + } + make_vortex_predicate(&pushed).transpose() }) .transpose() diff --git a/vortex-datafusion/tests/schema_evolution.rs b/vortex-datafusion/tests/schema_evolution.rs index 90d83eef978..5898211dbc9 100644 --- a/vortex-datafusion/tests/schema_evolution.rs +++ b/vortex-datafusion/tests/schema_evolution.rs @@ -15,19 +15,16 @@ use std::sync::LazyLock; use arrow_schema::DataType; use arrow_schema::Field; use arrow_schema::Schema; -use arrow_schema::SchemaRef; use datafusion::arrow::array::Array; use datafusion::arrow::array::ArrayRef as ArrowArrayRef; -use datafusion::arrow::array::Int32Array; use datafusion::arrow::array::RecordBatch; -use datafusion::arrow::array::StringViewArray; use datafusion::arrow::array::StructArray; -use datafusion::arrow::compute::concat_batches; use datafusion::datasource::listing::ListingOptions; use datafusion::datasource::listing::ListingTable; use datafusion::datasource::listing::ListingTableConfig; use datafusion::execution::SessionStateBuilder; use datafusion::execution::context::SessionContext; +use datafusion_common::assert_batches_eq; use datafusion_common::create_array; use datafusion_common::record_batch; use datafusion_datasource::ListingTableUrl; @@ -146,29 +143,17 @@ async fn test_filter_with_schema_evolution() { .collect() .await .unwrap(); - let table = concat_batches(&table_schema, result.iter()).unwrap(); - // We read back the full table, with nulls filled in for missing fields - assert_eq!( - table, - record_batch( - &table_schema, - vec![ - // a - Arc::new(StringViewArray::from(vec![ - Some("one"), - Some("two"), - Some("three"), - ])) as ArrowArrayRef, - // b - Arc::new(StringViewArray::from(vec![ - Option::<&str>::None, - None, - None - ])) as ArrowArrayRef, - ] - ) - ); + let expected = [ + "+-------+---+", + "| a | b |", + "+-------+---+", + "| one | |", + "| two | |", + "| three | |", + "+-------+---+", + ]; + assert_batches_eq!(expected, &result); } #[tokio::test] @@ -236,19 +221,16 @@ async fn test_filter_schema_evolution_order() { .collect() .await .unwrap(); - let result = concat_batches(&table_schema, result.iter()).unwrap(); - assert_eq!( - result, - record_batch( - &table_schema, - vec![ - // a - Arc::new(Int32Array::from(vec![Some(2)])) as ArrowArrayRef, - // b - Arc::new(StringViewArray::from(vec![Some("two"),])) as ArrowArrayRef, - ] - ) + assert_batches_eq!( + &[ + "+---+-----+", + "| a | b |", + "+---+-----+", + "| 2 | two |", + "+---+-----+", + ], + &result ); // Filter on the "a" column, which has different types for each file @@ -260,26 +242,22 @@ async fn test_filter_schema_evolution_order() { .collect() .await .unwrap(); - let table = concat_batches(&table_schema, result.iter()).unwrap(); + // let table = concat_batches(&table_schema, result.iter()).unwrap(); - // file1, then file2 - assert_eq!( - table, - record_batch( - &table_schema, - vec![ - // a field: present in both files - Arc::new(Int32Array::from(vec![Some(3), Some(5), Some(4), Some(6)])) - as ArrowArrayRef, - // b field: only present in file2, file1 fills with nulls - Arc::new(StringViewArray::from(vec![ - None, - None, - Some("four"), - Some("six") - ])) as ArrowArrayRef, - ] - ) + // a field: present in both files + // b field: only present in file2, file1 fills with nulls + assert_batches_eq!( + &[ + "+---+------+", + "| a | b |", + "+---+------+", + "| 3 | |", + "| 5 | |", + "| 4 | four |", + "| 6 | six |", + "+---+------+", + ], + &result ); } @@ -372,18 +350,24 @@ async fn test_filter_schema_evolution_struct_fields() { // Scan all the records, NULLs are filled in for nested optional fields. let full_scan = df.collect().await.unwrap(); - let full_scan = concat_batches(&table_schema, full_scan.iter()).unwrap(); - let expected = concat_batches( - &table_schema, + assert_batches_eq!( &[ - // host01 with extra nulls for the payload.instance field - make_metrics("host01.local", vec![1, 2, 3, 4], Some(vec![None; 4])), - host02, + "+--------------+-----------------------------+", + "| hostname | payload |", + "+--------------+-----------------------------+", + "| host01.local | {uptime: 1, instance: } |", + "| host01.local | {uptime: 2, instance: } |", + "| host01.local | {uptime: 3, instance: } |", + "| host01.local | {uptime: 4, instance: } |", + "| host02.local | {uptime: 10, instance: c6i} |", + "| host02.local | {uptime: 20, instance: c6i} |", + "| host02.local | {uptime: 30, instance: m5} |", + "| host02.local | {uptime: 40, instance: r5} |", + "+--------------+-----------------------------+", ], - ) - .unwrap(); - assert_eq!(full_scan, expected); + &full_scan + ); // run a filter that touches both the payload.uptime AND the payload.instance nested fields let df = ctx.read_table(table.clone()).unwrap(); @@ -400,25 +384,20 @@ async fn test_filter_schema_evolution_struct_fields() { .collect() .await .unwrap(); - let filtered_scan = concat_batches(&table_schema, filtered_scan.iter()).unwrap(); - let expected = concat_batches( - &table_schema, + + assert_batches_eq!( &[ - make_metrics("host01.local", vec![1, 2, 3, 4], Some(vec![None; 4])), - make_metrics( - "host02.local", - vec![10, 20], - Some(vec![Some("c6i"), Some("c6i")]), - ), + "+--------------+-----------------------------+", + "| hostname | payload |", + "+--------------+-----------------------------+", + "| host01.local | {uptime: 1, instance: } |", + "| host01.local | {uptime: 2, instance: } |", + "| host01.local | {uptime: 3, instance: } |", + "| host01.local | {uptime: 4, instance: } |", + "| host02.local | {uptime: 10, instance: c6i} |", + "| host02.local | {uptime: 20, instance: c6i} |", + "+--------------+-----------------------------+", ], - ) - .unwrap(); - assert_eq!(filtered_scan, expected); -} - -fn record_batch( - schema: &SchemaRef, - fields: impl IntoIterator, -) -> RecordBatch { - RecordBatch::try_new(schema.clone(), fields.into_iter().collect()).unwrap() + &filtered_scan + ); } From 6cbb597ff037e6fb01285c4562f9ddd6746cf552 Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Fri, 12 Dec 2025 15:03:12 -0500 Subject: [PATCH 7/8] modified upstream DF52 schema adapter backport Signed-off-by: Andrew Duffy --- vortex-datafusion/src/convert/exprs.rs | 55 --- vortex-datafusion/src/lib.rs | 1 + vortex-datafusion/src/persistent/opener.rs | 27 +- vortex-datafusion/src/persistent/source.rs | 24 +- vortex-datafusion/src/vendor/mod.rs | 7 + .../src/vendor/schema_rewriter.rs | 315 ++++++++++++++++++ vortex-datafusion/tests/schema_evolution.rs | 2 +- 7 files changed, 345 insertions(+), 86 deletions(-) create mode 100644 vortex-datafusion/src/vendor/mod.rs create mode 100644 vortex-datafusion/src/vendor/schema_rewriter.rs diff --git a/vortex-datafusion/src/convert/exprs.rs b/vortex-datafusion/src/convert/exprs.rs index c6270a23d49..3bab1edfa97 100644 --- a/vortex-datafusion/src/convert/exprs.rs +++ b/vortex-datafusion/src/convert/exprs.rs @@ -312,14 +312,10 @@ mod tests { use arrow_schema::DataType; use arrow_schema::Field; - use arrow_schema::Fields; use arrow_schema::Schema; use arrow_schema::TimeUnit as ArrowTimeUnit; - use datafusion::functions::core::getfield::GetFieldFunc; use datafusion_common::ScalarValue; - use datafusion_common::config::ConfigOptions; use datafusion_expr::Operator as DFOperator; - use datafusion_expr::ScalarUDF; use datafusion_physical_expr::PhysicalExpr; use datafusion_physical_plan::expressions as df_expr; use insta::assert_snapshot; @@ -620,55 +616,4 @@ mod tests { assert!(!can_be_pushed_down(&like_expr, &test_schema)); } - - #[test] - fn test_expr_from_df_get_field() { - let struct_col = Arc::new(df_expr::Column::new("my_struct", 0)) as Arc; - let field_name = Arc::new(df_expr::Literal::new(ScalarValue::Utf8(Some( - "field1".to_string(), - )))) as Arc; - let get_field_expr = ScalarFunctionExpr::new( - "get_field", - Arc::new(ScalarUDF::from(GetFieldFunc::new())), - vec![struct_col, field_name], - Arc::new(Field::new("field1", DataType::Utf8, true)), - Arc::new(ConfigOptions::new()), - ); - let result = Expression::try_from_df(&get_field_expr).unwrap(); - assert_snapshot!(result.display_tree().to_string(), @r#" - vortex.get_item "field1" - └── input: vortex.get_item "my_struct" - └── input: vortex.root - "#); - } - - #[rstest] - #[case::valid_field("field1", true)] - #[case::missing_field("nonexistent_field", false)] - fn test_can_be_pushed_down_get_field(#[case] field_name: &str, #[case] expected: bool) { - let struct_fields = Fields::from(vec![ - Field::new("field1", DataType::Utf8, true), - Field::new("field2", DataType::Int32, true), - ]); - let schema = Schema::new(vec![Field::new( - "my_struct", - DataType::Struct(struct_fields), - true, - )]); - - let struct_col = Arc::new(df_expr::Column::new("my_struct", 0)) as Arc; - let field_name_lit = Arc::new(df_expr::Literal::new(ScalarValue::Utf8(Some( - field_name.to_string(), - )))) as Arc; - - let get_field_expr = Arc::new(ScalarFunctionExpr::new( - "get_field", - Arc::new(ScalarUDF::from(GetFieldFunc::new())), - vec![struct_col, field_name_lit], - Arc::new(Field::new(field_name, DataType::Utf8, true)), - Arc::new(ConfigOptions::new()), - )) as Arc; - - assert_eq!(can_be_pushed_down(&get_field_expr, &schema), expected); - } } diff --git a/vortex-datafusion/src/lib.rs b/vortex-datafusion/src/lib.rs index 1acb75e07ef..fceff2d5462 100644 --- a/vortex-datafusion/src/lib.rs +++ b/vortex-datafusion/src/lib.rs @@ -10,6 +10,7 @@ use vortex::expr::stats::Precision; mod convert; mod persistent; +pub mod vendor; pub use persistent::*; diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index ca50664689d..32424c3b3b5 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -91,6 +91,7 @@ impl FileOpener for VortexOpener { let mut filter = self.filter.clone(); let file_pruning_predicate = self.file_pruning_predicate.clone(); let expr_adapter_factory = self.expr_adapter_factory.clone(); + let file_cache = self.file_cache.clone(); let table_schema = self.table_schema.clone(); let batch_size = self.batch_size; @@ -159,6 +160,12 @@ impl FileOpener for VortexOpener { if let Some(expr_adapter_factory) = expr_adapter_factory { // Replace column access for partition columns with literals + let partition_values = table_schema + .table_partition_cols() + .iter() + .cloned() + .zip(file.partition_values) + .collect(); // The adapter rewrites the expression to the local file schema, allowing // for schema evolution and divergence between the table's schema and individual files. @@ -169,14 +176,7 @@ impl FileOpener for VortexOpener { Arc::clone(table_schema.file_schema()), Arc::clone(&physical_file_schema), ) - .with_partition_values( - table_schema - .table_partition_cols() - .iter() - .cloned() - .zip(file.partition_values.iter().cloned()) - .collect(), - ) + .with_partition_values(partition_values) .rewrite(filter)?; // Expression might now reference columns that don't exist in the file, so we can give it @@ -264,10 +264,6 @@ impl FileOpener for VortexOpener { ))); } - for filter in pushed.iter() { - println!("- pushing: {filter}"); - } - make_vortex_predicate(&pushed).transpose() }) .transpose() @@ -327,7 +323,7 @@ impl FileOpener for VortexOpener { } } -/// If the file has a [`FileRange`](datafusion::datasource::listing::FileRange), we translate it into a row range in the file for the scan. +/// If the file has a [`FileRange`], we translate it into a row range in the file for the scan. fn apply_byte_range( file_range: FileRange, total_size: u64, @@ -386,6 +382,7 @@ mod tests { use vortex::session::VortexSession; use super::*; + use crate::vendor::schema_rewriter::DF52PhysicalExprAdapterFactory; static SESSION: LazyLock = LazyLock::new(VortexSession::default); @@ -477,7 +474,7 @@ mod tests { #[tokio::test] async fn test_open_with_adapter() -> anyhow::Result<()> { let expr_adapter_factory: Arc = - Arc::new(DefaultPhysicalExprAdapterFactory); + Arc::new(DF52PhysicalExprAdapterFactory); let object_store = Arc::new(InMemory::new()) as Arc; let file_path = "part=1/file.vortex"; @@ -780,7 +777,7 @@ mod tests { &col("my_struct").is_not_null(), table_schema.table_schema(), )), - Some(Arc::new(DefaultPhysicalExprAdapterFactory) as _), + Some(Arc::new(DF52PhysicalExprAdapterFactory) as _), ); // The opener should be able to open the file with a filter on the diff --git a/vortex-datafusion/src/persistent/source.rs b/vortex-datafusion/src/persistent/source.rs index fe09c5313ab..ac1ad37f03c 100644 --- a/vortex-datafusion/src/persistent/source.rs +++ b/vortex-datafusion/src/persistent/source.rs @@ -17,7 +17,6 @@ use datafusion_datasource::schema_adapter::DefaultSchemaAdapterFactory; use datafusion_datasource::schema_adapter::SchemaAdapterFactory; use datafusion_physical_expr::PhysicalExprRef; use datafusion_physical_expr::conjunction; -use datafusion_physical_expr_adapter::DefaultPhysicalExprAdapterFactory; use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory; use datafusion_physical_expr_common::physical_expr::fmt_sql; use datafusion_physical_plan::DisplayFormatType; @@ -39,6 +38,7 @@ use super::cache::VortexFileCache; use super::metrics::PARTITION_LABEL; use super::opener::VortexOpener; use crate::convert::exprs::can_be_pushed_down; +use crate::vendor::schema_rewriter::DF52PhysicalExprAdapterFactory; /// Execution plan for reading one or more Vortex files, intended to be consumed by [`DataSourceExec`]. /// @@ -81,19 +81,6 @@ impl VortexSource { layout_readers: Arc::new(DashMap::default()), } } - - /// Sets a [`PhysicalExprAdapterFactory`] for the [`VortexSource`]. - /// Currently, this must be provided in order to filter columns in files that have a different data type from the unified table schema. - /// - /// This factory will take precedence when opening files over instances provided by the [`FileScanConfig`]. - pub fn with_expr_adapter_factory( - &self, - expr_adapter_factory: Arc, - ) -> Arc { - let mut source = self.clone(); - source.expr_adapter_factory = Some(expr_adapter_factory); - Arc::new(source) - } } impl FileSource for VortexSource { @@ -116,6 +103,13 @@ impl FileSource for VortexSource { .expr_adapter_factory .as_ref() .or(base_config.expr_adapter_factory.as_ref()); + + if expr_adapter.is_some() { + tracing::warn!( + "Schema evolution with VortexSource may not work as expected if you override the adapter." + ); + } + let schema_adapter = self.schema_adapter_factory.as_ref(); // This match is here to support the behavior defined by [`ListingTable`], see https://github.com/apache/datafusion/issues/16800 for more details. @@ -132,7 +126,7 @@ impl FileSource for VortexSource { (None, schema_adapter.clone()) } (None, None) => ( - Some(Arc::new(DefaultPhysicalExprAdapterFactory) as _), + Some(Arc::new(DF52PhysicalExprAdapterFactory) as _), Arc::new(DefaultSchemaAdapterFactory) as _, ), }; diff --git a/vortex-datafusion/src/vendor/mod.rs b/vortex-datafusion/src/vendor/mod.rs new file mode 100644 index 00000000000..f673c9cbc5f --- /dev/null +++ b/vortex-datafusion/src/vendor/mod.rs @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! We are vendoring the physical expression adapter fixes for nested structs +//! that will be released in DF 52. + +pub mod schema_rewriter; diff --git a/vortex-datafusion/src/vendor/schema_rewriter.rs b/vortex-datafusion/src/vendor/schema_rewriter.rs new file mode 100644 index 00000000000..3d9f4ea4a93 --- /dev/null +++ b/vortex-datafusion/src/vendor/schema_rewriter.rs @@ -0,0 +1,315 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#![allow(missing_docs)] + +//! Physical expression schema rewriting utilities +//! +//! NOTE(aduffy): this is vendored until DF 52 is released, at which point this should +//! all be deleted. + +use std::sync::Arc; + +use datafusion_common::Result; +use datafusion_common::ScalarValue; +use datafusion_common::arrow::compute::can_cast_types; +use datafusion_common::arrow::datatypes::DataType; +use datafusion_common::arrow::datatypes::FieldRef; +use datafusion_common::arrow::datatypes::Schema; +use datafusion_common::arrow::datatypes::SchemaRef; +use datafusion_common::exec_err; +use datafusion_common::nested_struct::validate_struct_compatibility; +use datafusion_common::tree_node::Transformed; +use datafusion_common::tree_node::TransformedResult; +use datafusion_common::tree_node::TreeNode; +use datafusion_functions::core::getfield::GetFieldFunc; +use datafusion_physical_expr::ScalarFunctionExpr; +use datafusion_physical_expr::expressions::CastColumnExpr; +use datafusion_physical_expr::expressions::Column; +use datafusion_physical_expr::expressions::{self}; +use datafusion_physical_expr_adapter::PhysicalExprAdapter; +use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory; +use datafusion_physical_expr_common::physical_expr::PhysicalExpr; + +#[derive(Debug, Clone)] +pub struct DF52PhysicalExprAdapterFactory; + +impl PhysicalExprAdapterFactory for DF52PhysicalExprAdapterFactory { + fn create( + &self, + logical_file_schema: SchemaRef, + physical_file_schema: SchemaRef, + ) -> Arc { + Arc::new(DF52PhysicalExprAdapter { + logical_file_schema, + physical_file_schema, + partition_values: Vec::new(), + }) + } +} + +#[derive(Debug, Clone)] +pub struct DF52PhysicalExprAdapter { + logical_file_schema: SchemaRef, + physical_file_schema: SchemaRef, + partition_values: Vec<(FieldRef, ScalarValue)>, +} + +impl DF52PhysicalExprAdapter { + /// Create a new instance of the default physical expression adapter. + /// + /// This adapter rewrites expressions to match the physical schema of the file being scanned, + /// handling type mismatches and missing columns by filling them with default values. + pub fn new(logical_file_schema: SchemaRef, physical_file_schema: SchemaRef) -> Self { + Self { + logical_file_schema, + physical_file_schema, + partition_values: Vec::new(), + } + } +} + +impl PhysicalExprAdapter for DF52PhysicalExprAdapter { + fn rewrite(&self, expr: Arc) -> Result> { + let rewriter = DefaultPhysicalExprAdapterRewriter { + logical_file_schema: &self.logical_file_schema, + physical_file_schema: &self.physical_file_schema, + partition_fields: &self.partition_values, + }; + expr.transform(|expr| rewriter.rewrite_expr(Arc::clone(&expr))) + .data() + } + + fn with_partition_values( + &self, + partition_values: Vec<(FieldRef, ScalarValue)>, + ) -> Arc { + Arc::new(DF52PhysicalExprAdapter { + partition_values, + ..self.clone() + }) + } +} + +struct DefaultPhysicalExprAdapterRewriter<'a> { + logical_file_schema: &'a Schema, + physical_file_schema: &'a Schema, + partition_fields: &'a [(FieldRef, ScalarValue)], +} + +impl<'a> DefaultPhysicalExprAdapterRewriter<'a> { + fn rewrite_expr( + &self, + expr: Arc, + ) -> Result>> { + if let Some(transformed) = self.try_rewrite_struct_field_access(&expr)? { + return Ok(Transformed::yes(transformed)); + } + + if let Some(column) = expr.as_any().downcast_ref::() { + return self.rewrite_column(Arc::clone(&expr), column); + } + + Ok(Transformed::no(expr)) + } + + /// Attempt to rewrite struct field access expressions to return null if the field does not exist in the physical schema. + /// Note that this does *not* handle nested struct fields, only top-level struct field access. + /// See for more details. + fn try_rewrite_struct_field_access( + &self, + expr: &Arc, + ) -> Result>> { + let get_field_expr = + match ScalarFunctionExpr::try_downcast_func::(expr.as_ref()) { + Some(expr) => expr, + None => return Ok(None), + }; + + let source_expr = match get_field_expr.args().first() { + Some(expr) => expr, + None => return Ok(None), + }; + + let field_name_expr = match get_field_expr.args().get(1) { + Some(expr) => expr, + None => return Ok(None), + }; + + let lit = match field_name_expr + .as_any() + .downcast_ref::() + { + Some(lit) => lit, + None => return Ok(None), + }; + + let field_name = match lit.value().try_as_str().flatten() { + Some(name) => name, + None => return Ok(None), + }; + + let column = match source_expr.as_any().downcast_ref::() { + Some(column) => column, + None => return Ok(None), + }; + + let physical_field = match self.physical_file_schema.field_with_name(column.name()) { + Ok(field) => field, + Err(_) => return Ok(None), + }; + + let physical_struct_fields = match physical_field.data_type() { + DataType::Struct(fields) => fields, + _ => return Ok(None), + }; + + if physical_struct_fields + .iter() + .any(|f| f.name() == field_name) + { + return Ok(None); + } + + let logical_field = match self.logical_file_schema.field_with_name(column.name()) { + Ok(field) => field, + Err(_) => return Ok(None), + }; + + let logical_struct_fields = match logical_field.data_type() { + DataType::Struct(fields) => fields, + _ => return Ok(None), + }; + + let logical_struct_field = match logical_struct_fields + .iter() + .find(|f| f.name() == field_name) + { + Some(field) => field, + None => return Ok(None), + }; + + let null_value = ScalarValue::Null.cast_to(logical_struct_field.data_type())?; + Ok(Some(expressions::lit(null_value))) + } + + fn rewrite_column( + &self, + expr: Arc, + column: &Column, + ) -> Result>> { + // Get the logical field for this column if it exists in the logical schema + let logical_field = match self.logical_file_schema.field_with_name(column.name()) { + Ok(field) => field, + Err(e) => { + // If the column is a partition field, we can use the partition value + if let Some(partition_value) = self.get_partition_value(column.name()) { + return Ok(Transformed::yes(expressions::lit(partition_value))); + } + // This can be hit if a custom rewrite injected a reference to a column that doesn't exist in the logical schema. + // For example, a pre-computed column that is kept only in the physical schema. + // If the column exists in the physical schema, we can still use it. + if let Ok(physical_field) = self.physical_file_schema.field_with_name(column.name()) + { + // If the column exists in the physical schema, we can use it in place of the logical column. + // This is nice to users because if they do a rewrite that results in something like `physical_int32_col = 123u64` + // we'll at least handle the casts for them. + physical_field + } else { + // A completely unknown column that doesn't exist in either schema! + // This should probably never be hit unless something upstream broke, but nonetheless it's better + // for us to return a handleable error than to panic / do something unexpected. + return Err(e.into()); + } + } + }; + + // Check if the column exists in the physical schema + let physical_column_index = match self.physical_file_schema.index_of(column.name()) { + Ok(index) => index, + Err(_) => { + if !logical_field.is_nullable() { + return exec_err!( + "Non-nullable column '{}' is missing from the physical schema", + column.name() + ); + } + // If the column is missing from the physical schema fill it in with nulls as `SchemaAdapter` would do. + // TODO: do we need to sync this with what the `SchemaAdapter` actually does? + // While the default implementation fills in nulls in theory a custom `SchemaAdapter` could do something else! + // See https://github.com/apache/datafusion/issues/16527 + let null_value = ScalarValue::Null.cast_to(logical_field.data_type())?; + return Ok(Transformed::yes(expressions::lit(null_value))); + } + }; + let physical_field = self.physical_file_schema.field(physical_column_index); + + let column = match ( + column.index() == physical_column_index, + logical_field.data_type() == physical_field.data_type(), + ) { + // If the column index matches and the data types match, we can use the column as is + (true, true) => return Ok(Transformed::no(expr)), + // If the indexes or data types do not match, we need to create a new column expression + (true, _) => column.clone(), + (false, _) => Column::new_with_schema(logical_field.name(), self.physical_file_schema)?, + }; + + if logical_field.data_type() == physical_field.data_type() { + // If the data types match, we can use the column as is + return Ok(Transformed::yes(Arc::new(column))); + } + + // We need to cast the column to the logical data type + // TODO: add optimization to move the cast from the column to literal expressions in the case of `col = 123` + // since that's much cheaper to evalaute. + // See https://github.com/apache/datafusion/issues/15780#issuecomment-2824716928 + match (physical_field.data_type(), logical_field.data_type()) { + (DataType::Struct(physical_fields), DataType::Struct(logical_fields)) => { + validate_struct_compatibility(physical_fields, logical_fields)?; + } + _ => { + let is_compatible = + can_cast_types(physical_field.data_type(), logical_field.data_type()); + if !is_compatible { + return exec_err!( + "Cannot cast column '{}' from '{}' (physical data type) to '{}' (logical data type)", + column.name(), + physical_field.data_type(), + logical_field.data_type() + ); + } + } + } + + let cast_expr = Arc::new(CastColumnExpr::new( + Arc::new(column), + Arc::new(physical_field.clone()), + Arc::new(logical_field.clone()), + None, + )); + + Ok(Transformed::yes(cast_expr)) + } + + fn get_partition_value(&self, column_name: &str) -> Option { + self.partition_fields + .iter() + .find(|(field, _)| field.name() == column_name) + .map(|(_, value)| value.clone()) + } +} diff --git a/vortex-datafusion/tests/schema_evolution.rs b/vortex-datafusion/tests/schema_evolution.rs index 5898211dbc9..637893bf73e 100644 --- a/vortex-datafusion/tests/schema_evolution.rs +++ b/vortex-datafusion/tests/schema_evolution.rs @@ -335,7 +335,7 @@ async fn test_filter_schema_evolution_struct_fields() { let table = ListingTable::try_new( ListingTableConfig::new(table_url) .with_listing_options(list_opts) - .with_schema(read_schema.clone()), + .with_schema(read_schema.clone()), // .with_expr_adapter_factory(Arc::new(DF52PhysicalExprAdapterFactory)), ) .unwrap(); From 78e953e41a222f22cac9c1b91cad671649597619 Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Fri, 12 Dec 2025 15:10:47 -0500 Subject: [PATCH 8/8] remove struct test Signed-off-by: Andrew Duffy --- .../src/arrays/struct_/compute/mod.rs | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/vortex-array/src/arrays/struct_/compute/mod.rs b/vortex-array/src/arrays/struct_/compute/mod.rs index a34bf99b456..2b0827158ba 100644 --- a/vortex-array/src/arrays/struct_/compute/mod.rs +++ b/vortex-array/src/arrays/struct_/compute/mod.rs @@ -199,41 +199,6 @@ mod tests { assert_eq!(casted.dtype(), &nullable_dtype); } - #[test] - fn test_cast_cannot_change_name_order() { - let array = StructArray::try_new( - ["xs", "ys", "zs"].into(), - vec![ - buffer![1u8].into_array(), - buffer![1u8].into_array(), - buffer![1u8].into_array(), - ], - 1, - Validity::NonNullable, - ) - .unwrap(); - - let tu8 = DType::Primitive(PType::U8, NonNullable); - - let result = cast( - array.as_ref(), - &DType::Struct( - StructFields::new( - FieldNames::from(["ys", "xs", "zs"]), - vec![tu8.clone(), tu8.clone(), tu8], - ), - NonNullable, - ), - ); - assert!( - result.as_ref().is_err_and(|err| { - err.to_string() - .contains("cannot cast {xs=u8, ys=u8, zs=u8} to {ys=u8, xs=u8, zs=u8}") - }), - "{result:?}" - ); - } - #[test] fn test_cast_complex_struct() { let xs = PrimitiveArray::from_option_iter([Some(0i64), Some(1), Some(2), Some(3), Some(4)]);