From 1142d507d5b07bb031b9914b9bb7317a6929a2f5 Mon Sep 17 00:00:00 2001 From: nuno-faria Date: Tue, 5 Aug 2025 10:43:59 +0100 Subject: [PATCH 1/3] chore: Upgrade datafusion to version 49 --- Cargo.lock | 194 +++++++++++-------- Cargo.toml | 10 +- examples/datafusion-ffi-example/Cargo.lock | 209 ++++++++++++++------- examples/datafusion-ffi-example/Cargo.toml | 4 +- python/tests/test_dataframe.py | 12 +- python/tests/test_expr.py | 2 +- python/tests/test_functions.py | 3 +- src/expr.rs | 6 +- src/expr/copy_to.rs | 8 +- src/expr/join.rs | 10 +- src/expr/literal.rs | 10 +- src/functions.rs | 6 +- 12 files changed, 299 insertions(+), 175 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2622ea20..67986df9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -307,6 +307,7 @@ dependencies = [ "arrow-schema", "flatbuffers", "lz4_flex", + "zstd", ] [[package]] @@ -628,6 +629,15 @@ dependencies = [ "bzip2-sys", ] +[[package]] +name = "bzip2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea8dcd42434048e4f7a304411d9273a411f647446c1234a65ce0554923f4cff" +dependencies = [ + "libbz2-rs-sys", +] + [[package]] name = "bzip2-sys" version = "0.1.13+1.0.8" @@ -878,16 +888,16 @@ dependencies = [ [[package]] name = "datafusion" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a11e19a7ccc5bb979c95c1dceef663eab39c9061b3bbf8d1937faf0f03bf41f" +checksum = "0f47772c28553d837e12cdcc0fb04c2a0fe8eca8b704a30f721d076f32407435" dependencies = [ "arrow", "arrow-ipc", "arrow-schema", "async-trait", "bytes", - "bzip2 0.5.2", + "bzip2 0.6.0", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -915,6 +925,7 @@ dependencies = [ "datafusion-sql", "flate2", "futures", + "hex", "itertools 0.14.0", "log", "object_store", @@ -933,9 +944,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94985e67cab97b1099db2a7af11f31a45008b282aba921c1e1d35327c212ec18" +checksum = "4b6b29c9c922959285fac53139e12c81014e2ca54704f20355edd7e9d11fd773" dependencies = [ "arrow", "async-trait", @@ -959,9 +970,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e002df133bdb7b0b9b429d89a69aa77b35caeadee4498b2ce1c7c23a99516988" +checksum = "7313553e4c01d184dd49183afdfa22f23204a10a26dd12e6f799203d8fdb95c2" dependencies = [ "arrow", "async-trait", @@ -982,17 +993,19 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13242fc58fd753787b0a538e5ae77d356cb9d0656fa85a591a33c5f106267f6" +checksum = "3d66104731b7476a8c86fbe7a6fd741e6329791166ac89a91fcd8336a560ddaf" dependencies = [ "ahash", "apache-avro", "arrow", "arrow-ipc", "base64 0.22.1", + "chrono", "half", "hashbrown 0.14.5", + "hex", "indexmap", "libc", "log", @@ -1007,9 +1020,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2239f964e95c3a5d6b4a8cde07e646de8995c1396a7fd62c6e784f5341db499" +checksum = "0e7527ecdfeae6961a8564d3b036507a67bd467fd36a9f10cf8ad7a99db1f1bc" dependencies = [ "futures", "log", @@ -1018,15 +1031,15 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cf792579bc8bf07d1b2f68c2d5382f8a63679cce8fbebfd4ba95742b6e08864" +checksum = "40e5076be33d8eb9f4d99858e5f3477b36c07e61eee8eb93c4320428d9e1e344" dependencies = [ "arrow", "async-compression", "async-trait", "bytes", - "bzip2 0.5.2", + "bzip2 0.6.0", "chrono", "datafusion-common", "datafusion-common-runtime", @@ -1054,9 +1067,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4de733d231abb0fba663ff60fd37bf7171fa8b2e46e8a99e41362001821d116e" +checksum = "831cfe556658133ea4270d616164ce27f737e9e4d5e359e1b1b269e0bf767cef" dependencies = [ "apache-avro", "arrow", @@ -1079,9 +1092,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc114f9a1415174f3e8d2719c371fc72092ef2195a7955404cfe6b2ba29a706" +checksum = "785518d0f2f136c19b9389a10762c01a5aeb5fcdebdb244297bb656b2862dc88" dependencies = [ "arrow", "async-trait", @@ -1104,9 +1117,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d88dd5e215c420a52362b9988ecd4cefd71081b730663d4f7d886f706111fc75" +checksum = "71cb7c3bad0951bf5c52505d0e6d87e6c0098156d2a195924cbcdc82238d29ba" dependencies = [ "arrow", "async-trait", @@ -1129,9 +1142,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33692acdd1fbe75280d14f4676fe43f39e9cb36296df56575aa2cac9a819e4cf" +checksum = "ea76ad2c5189c98a6b1d4bdf6c3b3caacc9701c417af6661597c946a201bc328" dependencies = [ "arrow", "async-trait", @@ -1147,8 +1160,10 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-optimizer", "datafusion-physical-plan", + "datafusion-pruning", "datafusion-session", "futures", + "hex", "itertools 0.14.0", "log", "object_store", @@ -1160,15 +1175,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0e7b648387b0c1937b83cb328533c06c923799e73a9e3750b762667f32662c0" +checksum = "6bcc45e380db5c6033c3f39e765a3d752679f14315060a7f4030a60066a36946" [[package]] name = "datafusion-execution" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9609d83d52ff8315283c6dad3b97566e877d8f366fab4c3297742f33dcd636c7" +checksum = "8209805fdce3d5c6e1625f674d3e4ce93e995a56d3709a0bb8d4361062652596" dependencies = [ "arrow", "dashmap", @@ -1185,11 +1200,12 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e75230cd67f650ef0399eb00f54d4a073698f2c0262948298e5299fc7324da63" +checksum = "7879a845e72a00cacffacbdf5f40626049cb9584d2ba8aa0b9172f09833110ab" dependencies = [ "arrow", + "async-trait", "chrono", "datafusion-common", "datafusion-doc", @@ -1206,9 +1222,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70fafb3a045ed6c49cfca0cd090f62cf871ca6326cc3355cb0aaf1260fa760b6" +checksum = "6da7e47e70ef2c7678735c82c392bd74687004043f5fc8072ab8678dc6fa459d" dependencies = [ "arrow", "datafusion-common", @@ -1219,9 +1235,9 @@ dependencies = [ [[package]] name = "datafusion-ffi" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "257de77f836e9bad3e0274d4ffc88555ebf559ae30d4e9c674b9809104c1cc3b" +checksum = "ba959e6fd3a9a503201600e5575b3cb2aaa066b876317218ce25f0c0f836ac5f" dependencies = [ "abi_stable", "arrow", @@ -1241,9 +1257,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf9a9cf655265861a20453b1e58357147eab59bdc90ce7f2f68f1f35104d3bb" +checksum = "5e7b92b04c5c3b1151f055251b36e272071f9088d9701826a533cb4f764af1c8" dependencies = [ "arrow", "arrow-buffer", @@ -1270,9 +1286,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f07e49733d847be0a05235e17b884d326a2fd402c97a89fe8bcf0bfba310005" +checksum = "3f16cb922b62e535a4d484961ac2c1c6d188dbe02e85e026c05f0fabbc8f814e" dependencies = [ "ahash", "arrow", @@ -1291,9 +1307,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4512607e10d72b0b0a1dc08f42cb5bd5284cb8348b7fea49dc83409493e32b1b" +checksum = "6f71bb59dc8b4dc985c911f2e0d8cf426c21f565b56dca4b852c244101a1a7a2" dependencies = [ "ahash", "arrow", @@ -1304,9 +1320,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab331806e34f5545e5f03396e4d5068077395b1665795d8f88c14ec4f1e0b7a" +checksum = "27eb3b98a2eb02a8af4ef19cc793cac21fc98d8720b987f15d7d25b8cc875f4d" dependencies = [ "arrow", "arrow-ord", @@ -1316,6 +1332,7 @@ dependencies = [ "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", "itertools 0.14.0", @@ -1325,9 +1342,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4ac2c0be983a06950ef077e34e0174aa0cb9e346f3aeae459823158037ade37" +checksum = "350e0940fc3e2fa4645a4d323f9ebf9258b2d7fdad12013a471cae4ae5568683" dependencies = [ "arrow", "async-trait", @@ -1341,9 +1358,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f3d92731de384c90906941d36dcadf6a86d4128409a9c5cd916662baed5f53" +checksum = "df03c6c62039578fd110b327c474846fdf3d9077a568f1e8706e585ed30cb98d" dependencies = [ "arrow", "datafusion-common", @@ -1359,9 +1376,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c679f8bf0971704ec8fd4249fcbb2eb49d6a12cc3e7a840ac047b4928d3541b5" +checksum = "083659a95914bf3ca568a72b085cb8654576fef1236b260dc2379cb8e5f922b2" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1369,9 +1386,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2821de7cb0362d12e75a5196b636a59ea3584ec1e1cc7dc6f5e34b9e8389d251" +checksum = "4cabe1f32daa2fa54e6b20d14a13a9e85bef97c4161fe8a90d76b6d9693a5ac4" dependencies = [ "datafusion-expr", "quote", @@ -1380,14 +1397,15 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1594c7a97219ede334f25347ad8d57056621e7f4f35a0693c8da876e10dd6a53" +checksum = "6e12a97dcb0ccc569798be1289c744829cce5f18cc9b037054f8d7f93e1d57be" dependencies = [ "arrow", "chrono", "datafusion-common", "datafusion-expr", + "datafusion-expr-common", "datafusion-physical-expr", "indexmap", "itertools 0.14.0", @@ -1399,9 +1417,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6da0f2412088d23f6b01929dedd687b5aee63b19b674eb73d00c3eb3c883b7" +checksum = "41312712b8659a82b4e9faa8d97a018e7f2ccbdedf2f7cb93ecf256e39858c86" dependencies = [ "ahash", "arrow", @@ -1421,9 +1439,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb0dbd9213078a593c3fe28783beaa625a4e6c6a6c797856ee2ba234311fb96" +checksum = "be1649a60ea0319496d616ae3554e84dfcc262c201ab4439abcd83cca989b85b" dependencies = [ "ahash", "arrow", @@ -1435,9 +1453,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d140854b2db3ef8ac611caad12bfb2e1e1de827077429322a6188f18fc0026a" +checksum = "ea3f5b8ba6122426774aaaf11325740b8e5d3afaab9ab39dc63423adca554748" dependencies = [ "arrow", "datafusion-common", @@ -1447,6 +1465,7 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-pruning", "itertools 0.14.0", "log", "recursive", @@ -1454,9 +1473,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b46cbdf21a01206be76d467f325273b22c559c744a012ead5018dfe79597de08" +checksum = "6a595f296929d6cffa12b993ea53e9fe8215fada050d78626c5cf0e2f02b0205" dependencies = [ "ahash", "arrow", @@ -1484,9 +1503,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3fc7a2744332c2ef8804274c21f9fa664b4ca5889169250a6fd6b649ee5d16c" +checksum = "f4ab6f4fa0f3bbfbc0b4f89485bcd7cbed6cca0347e8d1eda50b66b779725b6e" dependencies = [ "arrow", "chrono", @@ -1500,18 +1519,36 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "800add86852f12e3d249867425de2224c1e9fb7adc2930460548868781fbeded" +checksum = "5ba94d76d85459ebbf7c29aa1b001234d551e840192c742a4237ec22de45a557" dependencies = [ "arrow", "datafusion-common", "prost", ] +[[package]] +name = "datafusion-pruning" +version = "49.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "391a457b9d23744c53eeb89edd1027424cba100581488d89800ed841182df905" +dependencies = [ + "arrow", + "arrow-schema", + "datafusion-common", + "datafusion-datasource", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "itertools 0.14.0", + "log", +] + [[package]] name = "datafusion-python" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "async-trait", @@ -1536,9 +1573,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a72733766ddb5b41534910926e8da5836622316f6283307fd9fb7e19811a59c" +checksum = "dd5f2fe790f43839c70fb9604c4f9b59ad290ef64e1d2f927925dd34a9245406" dependencies = [ "arrow", "async-trait", @@ -1560,9 +1597,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5162338cdec9cc7ea13a0e6015c361acad5ec1d88d83f7c86301f789473971f" +checksum = "2ebebb82fda37f62f06fe14339f4faa9f197a0320cc4d26ce2a5fd53a5ccd27c" dependencies = [ "arrow", "bigdecimal", @@ -1577,9 +1614,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af6029c08002772fb2c38a191d21a8b03af78750f42d9b896222f6b0e16c84cf" +checksum = "1d5a2bb4746c340a59cb9bdb4728826fff71d116a2a8e1c235f0988def3f5fdc" dependencies = [ "async-recursion", "async-trait", @@ -2303,6 +2340,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "libbz2-rs-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775bf80d5878ab7c2b1080b5351a48b2f737d9f6f8b383574eebcc22be0dfccb" + [[package]] name = "libc" version = "0.2.174" @@ -2568,9 +2611,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781f96d79ed0f961a7021424ab01840efbda64ae7a505aaea195efc91eaaec4" +checksum = "efc4f07659e11cd45a341cd24d71e683e3be65d9ff1f8150061678fe60437496" dependencies = [ "async-trait", "base64 0.22.1", @@ -2675,6 +2718,7 @@ dependencies = [ "num-bigint", "object_store", "paste", + "ring", "seq-macro", "simdutf8", "snap", @@ -3028,9 +3072,9 @@ checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" [[package]] name = "quick-xml" -version = "0.37.5" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +checksum = "8927b0664f5c5a98265138b7e3f90aa19a6b21353182469ace36d4ac527b7b1b" dependencies = [ "memchr", "serde", @@ -3712,9 +3756,9 @@ dependencies = [ [[package]] name = "substrait" -version = "0.56.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13de2e20128f2a018dab1cfa30be83ae069219a65968c6f89df66ad124de2397" +checksum = "de6d24c270c6c672a86c183c3a8439ba46c1936f93cf7296aa692de3b0ff0228" dependencies = [ "heck", "pbjson", diff --git a/Cargo.toml b/Cargo.toml index b66c5330..8d8d3a5c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "48.0.0" +version = "49.0.0" homepage = "https://datafusion.apache.org/python" repository = "https://github.com/apache/datafusion-python" authors = ["Apache DataFusion "] @@ -39,10 +39,10 @@ pyo3 = { version = "0.24", features = ["extension-module", "abi3", "abi3-py39"] pyo3-async-runtimes = { version = "0.24", features = ["tokio-runtime"]} pyo3-log = "0.12.4" arrow = { version = "55.1.0", features = ["pyarrow"] } -datafusion = { version = "48.0.0", features = ["avro", "unicode_expressions"] } -datafusion-substrait = { version = "48.0.0", optional = true } -datafusion-proto = { version = "48.0.0" } -datafusion-ffi = { version = "48.0.0" } +datafusion = { version = "49.0.0", features = ["avro", "unicode_expressions"] } +datafusion-substrait = { version = "49.0.0", optional = true } +datafusion-proto = { version = "49.0.0" } +datafusion-ffi = { version = "49.0.0" } prost = "0.13.1" # keep in line with `datafusion-substrait` uuid = { version = "1.16", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } diff --git a/examples/datafusion-ffi-example/Cargo.lock b/examples/datafusion-ffi-example/Cargo.lock index 75201763..e2ebc692 100644 --- a/examples/datafusion-ffi-example/Cargo.lock +++ b/examples/datafusion-ffi-example/Cargo.lock @@ -265,6 +265,7 @@ dependencies = [ "arrow-schema", "flatbuffers", "lz4_flex", + "zstd", ] [[package]] @@ -375,7 +376,7 @@ version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" dependencies = [ - "bzip2", + "bzip2 0.5.2", "flate2", "futures-core", "memchr", @@ -540,6 +541,15 @@ dependencies = [ "bzip2-sys", ] +[[package]] +name = "bzip2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea8dcd42434048e4f7a304411d9273a411f647446c1234a65ce0554923f4cff" +dependencies = [ + "libbz2-rs-sys", +] + [[package]] name = "bzip2-sys" version = "0.1.13+1.0.8" @@ -749,16 +759,16 @@ dependencies = [ [[package]] name = "datafusion" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a11e19a7ccc5bb979c95c1dceef663eab39c9061b3bbf8d1937faf0f03bf41f" +checksum = "0f47772c28553d837e12cdcc0fb04c2a0fe8eca8b704a30f721d076f32407435" dependencies = [ "arrow", "arrow-ipc", "arrow-schema", "async-trait", "bytes", - "bzip2", + "bzip2 0.6.0", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -785,6 +795,7 @@ dependencies = [ "datafusion-sql", "flate2", "futures", + "hex", "itertools", "log", "object_store", @@ -803,9 +814,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94985e67cab97b1099db2a7af11f31a45008b282aba921c1e1d35327c212ec18" +checksum = "4b6b29c9c922959285fac53139e12c81014e2ca54704f20355edd7e9d11fd773" dependencies = [ "arrow", "async-trait", @@ -829,9 +840,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e002df133bdb7b0b9b429d89a69aa77b35caeadee4498b2ce1c7c23a99516988" +checksum = "7313553e4c01d184dd49183afdfa22f23204a10a26dd12e6f799203d8fdb95c2" dependencies = [ "arrow", "async-trait", @@ -852,16 +863,18 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13242fc58fd753787b0a538e5ae77d356cb9d0656fa85a591a33c5f106267f6" +checksum = "3d66104731b7476a8c86fbe7a6fd741e6329791166ac89a91fcd8336a560ddaf" dependencies = [ "ahash", "arrow", "arrow-ipc", "base64", + "chrono", "half", "hashbrown 0.14.5", + "hex", "indexmap", "libc", "log", @@ -876,9 +889,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2239f964e95c3a5d6b4a8cde07e646de8995c1396a7fd62c6e784f5341db499" +checksum = "0e7527ecdfeae6961a8564d3b036507a67bd467fd36a9f10cf8ad7a99db1f1bc" dependencies = [ "futures", "log", @@ -887,15 +900,15 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cf792579bc8bf07d1b2f68c2d5382f8a63679cce8fbebfd4ba95742b6e08864" +checksum = "40e5076be33d8eb9f4d99858e5f3477b36c07e61eee8eb93c4320428d9e1e344" dependencies = [ "arrow", "async-compression", "async-trait", "bytes", - "bzip2", + "bzip2 0.6.0", "chrono", "datafusion-common", "datafusion-common-runtime", @@ -923,9 +936,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc114f9a1415174f3e8d2719c371fc72092ef2195a7955404cfe6b2ba29a706" +checksum = "785518d0f2f136c19b9389a10762c01a5aeb5fcdebdb244297bb656b2862dc88" dependencies = [ "arrow", "async-trait", @@ -948,9 +961,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d88dd5e215c420a52362b9988ecd4cefd71081b730663d4f7d886f706111fc75" +checksum = "71cb7c3bad0951bf5c52505d0e6d87e6c0098156d2a195924cbcdc82238d29ba" dependencies = [ "arrow", "async-trait", @@ -973,9 +986,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33692acdd1fbe75280d14f4676fe43f39e9cb36296df56575aa2cac9a819e4cf" +checksum = "ea76ad2c5189c98a6b1d4bdf6c3b3caacc9701c417af6661597c946a201bc328" dependencies = [ "arrow", "async-trait", @@ -991,8 +1004,10 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-optimizer", "datafusion-physical-plan", + "datafusion-pruning", "datafusion-session", "futures", + "hex", "itertools", "log", "object_store", @@ -1004,15 +1019,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0e7b648387b0c1937b83cb328533c06c923799e73a9e3750b762667f32662c0" +checksum = "6bcc45e380db5c6033c3f39e765a3d752679f14315060a7f4030a60066a36946" [[package]] name = "datafusion-execution" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9609d83d52ff8315283c6dad3b97566e877d8f366fab4c3297742f33dcd636c7" +checksum = "8209805fdce3d5c6e1625f674d3e4ce93e995a56d3709a0bb8d4361062652596" dependencies = [ "arrow", "dashmap", @@ -1029,11 +1044,12 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e75230cd67f650ef0399eb00f54d4a073698f2c0262948298e5299fc7324da63" +checksum = "7879a845e72a00cacffacbdf5f40626049cb9584d2ba8aa0b9172f09833110ab" dependencies = [ "arrow", + "async-trait", "chrono", "datafusion-common", "datafusion-doc", @@ -1050,9 +1066,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70fafb3a045ed6c49cfca0cd090f62cf871ca6326cc3355cb0aaf1260fa760b6" +checksum = "6da7e47e70ef2c7678735c82c392bd74687004043f5fc8072ab8678dc6fa459d" dependencies = [ "arrow", "datafusion-common", @@ -1063,9 +1079,9 @@ dependencies = [ [[package]] name = "datafusion-ffi" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "257de77f836e9bad3e0274d4ffc88555ebf559ae30d4e9c674b9809104c1cc3b" +checksum = "ba959e6fd3a9a503201600e5575b3cb2aaa066b876317218ce25f0c0f836ac5f" dependencies = [ "abi_stable", "arrow", @@ -1099,9 +1115,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf9a9cf655265861a20453b1e58357147eab59bdc90ce7f2f68f1f35104d3bb" +checksum = "5e7b92b04c5c3b1151f055251b36e272071f9088d9701826a533cb4f764af1c8" dependencies = [ "arrow", "arrow-buffer", @@ -1128,9 +1144,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f07e49733d847be0a05235e17b884d326a2fd402c97a89fe8bcf0bfba310005" +checksum = "3f16cb922b62e535a4d484961ac2c1c6d188dbe02e85e026c05f0fabbc8f814e" dependencies = [ "ahash", "arrow", @@ -1149,9 +1165,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4512607e10d72b0b0a1dc08f42cb5bd5284cb8348b7fea49dc83409493e32b1b" +checksum = "6f71bb59dc8b4dc985c911f2e0d8cf426c21f565b56dca4b852c244101a1a7a2" dependencies = [ "ahash", "arrow", @@ -1162,9 +1178,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab331806e34f5545e5f03396e4d5068077395b1665795d8f88c14ec4f1e0b7a" +checksum = "27eb3b98a2eb02a8af4ef19cc793cac21fc98d8720b987f15d7d25b8cc875f4d" dependencies = [ "arrow", "arrow-ord", @@ -1174,6 +1190,7 @@ dependencies = [ "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", "itertools", @@ -1183,9 +1200,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4ac2c0be983a06950ef077e34e0174aa0cb9e346f3aeae459823158037ade37" +checksum = "350e0940fc3e2fa4645a4d323f9ebf9258b2d7fdad12013a471cae4ae5568683" dependencies = [ "arrow", "async-trait", @@ -1199,9 +1216,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f3d92731de384c90906941d36dcadf6a86d4128409a9c5cd916662baed5f53" +checksum = "df03c6c62039578fd110b327c474846fdf3d9077a568f1e8706e585ed30cb98d" dependencies = [ "arrow", "datafusion-common", @@ -1217,9 +1234,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c679f8bf0971704ec8fd4249fcbb2eb49d6a12cc3e7a840ac047b4928d3541b5" +checksum = "083659a95914bf3ca568a72b085cb8654576fef1236b260dc2379cb8e5f922b2" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1227,9 +1244,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2821de7cb0362d12e75a5196b636a59ea3584ec1e1cc7dc6f5e34b9e8389d251" +checksum = "4cabe1f32daa2fa54e6b20d14a13a9e85bef97c4161fe8a90d76b6d9693a5ac4" dependencies = [ "datafusion-expr", "quote", @@ -1238,14 +1255,15 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1594c7a97219ede334f25347ad8d57056621e7f4f35a0693c8da876e10dd6a53" +checksum = "6e12a97dcb0ccc569798be1289c744829cce5f18cc9b037054f8d7f93e1d57be" dependencies = [ "arrow", "chrono", "datafusion-common", "datafusion-expr", + "datafusion-expr-common", "datafusion-physical-expr", "indexmap", "itertools", @@ -1257,9 +1275,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6da0f2412088d23f6b01929dedd687b5aee63b19b674eb73d00c3eb3c883b7" +checksum = "41312712b8659a82b4e9faa8d97a018e7f2ccbdedf2f7cb93ecf256e39858c86" dependencies = [ "ahash", "arrow", @@ -1279,9 +1297,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb0dbd9213078a593c3fe28783beaa625a4e6c6a6c797856ee2ba234311fb96" +checksum = "be1649a60ea0319496d616ae3554e84dfcc262c201ab4439abcd83cca989b85b" dependencies = [ "ahash", "arrow", @@ -1293,9 +1311,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d140854b2db3ef8ac611caad12bfb2e1e1de827077429322a6188f18fc0026a" +checksum = "ea3f5b8ba6122426774aaaf11325740b8e5d3afaab9ab39dc63423adca554748" dependencies = [ "arrow", "datafusion-common", @@ -1305,6 +1323,7 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-pruning", "itertools", "log", "recursive", @@ -1312,9 +1331,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b46cbdf21a01206be76d467f325273b22c559c744a012ead5018dfe79597de08" +checksum = "6a595f296929d6cffa12b993ea53e9fe8215fada050d78626c5cf0e2f02b0205" dependencies = [ "ahash", "arrow", @@ -1342,9 +1361,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3fc7a2744332c2ef8804274c21f9fa664b4ca5889169250a6fd6b649ee5d16c" +checksum = "f4ab6f4fa0f3bbfbc0b4f89485bcd7cbed6cca0347e8d1eda50b66b779725b6e" dependencies = [ "arrow", "chrono", @@ -1358,20 +1377,38 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "800add86852f12e3d249867425de2224c1e9fb7adc2930460548868781fbeded" +checksum = "5ba94d76d85459ebbf7c29aa1b001234d551e840192c742a4237ec22de45a557" dependencies = [ "arrow", "datafusion-common", "prost", ] +[[package]] +name = "datafusion-pruning" +version = "49.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "391a457b9d23744c53eeb89edd1027424cba100581488d89800ed841182df905" +dependencies = [ + "arrow", + "arrow-schema", + "datafusion-common", + "datafusion-datasource", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "itertools", + "log", +] + [[package]] name = "datafusion-session" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a72733766ddb5b41534910926e8da5836622316f6283307fd9fb7e19811a59c" +checksum = "dd5f2fe790f43839c70fb9604c4f9b59ad290ef64e1d2f927925dd34a9245406" dependencies = [ "arrow", "async-trait", @@ -1393,9 +1430,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5162338cdec9cc7ea13a0e6015c361acad5ec1d88d83f7c86301f789473971f" +checksum = "2ebebb82fda37f62f06fe14339f4faa9f197a0320cc4d26ce2a5fd53a5ccd27c" dependencies = [ "arrow", "bigdecimal", @@ -1621,8 +1658,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.1+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -1973,6 +2012,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "libbz2-rs-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775bf80d5878ab7c2b1080b5351a48b2f737d9f6f8b383574eebcc22be0dfccb" + [[package]] name = "libc" version = "0.2.174" @@ -2182,9 +2227,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781f96d79ed0f961a7021424ab01840efbda64ae7a505aaea195efc91eaaec4" +checksum = "efc4f07659e11cd45a341cd24d71e683e3be65d9ff1f8150061678fe60437496" dependencies = [ "async-trait", "bytes", @@ -2269,6 +2314,7 @@ dependencies = [ "num-bigint", "object_store", "paste", + "ring", "seq-macro", "simdutf8", "snap", @@ -2621,6 +2667,20 @@ dependencies = [ "tstr", ] +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rustc-demangle" version = "0.1.25" @@ -3050,6 +3110,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.4" @@ -3290,6 +3356,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.59.0" diff --git a/examples/datafusion-ffi-example/Cargo.toml b/examples/datafusion-ffi-example/Cargo.toml index b26ab48e..65a58858 100644 --- a/examples/datafusion-ffi-example/Cargo.toml +++ b/examples/datafusion-ffi-example/Cargo.toml @@ -21,8 +21,8 @@ version = "0.2.0" edition = "2021" [dependencies] -datafusion = { version = "48.0.0" } -datafusion-ffi = { version = "48.0.0" } +datafusion = { version = "49.0.0" } +datafusion-ffi = { version = "49.0.0" } pyo3 = { version = "0.23", features = ["extension-module", "abi3", "abi3-py39"] } arrow = { version = "55.0.0" } arrow-array = { version = "55.0.0" } diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index a3870ead..590bad32 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -1796,15 +1796,15 @@ def test_write_parquet_with_options_max_row_group_size( """Test configuring the max number of rows per group in Parquet. These test cases guarantee that the number of rows for each row group is max_row_group_size, given the total number of rows is a multiple of max_row_group_size.""" + path = f"{tmp_path}/t.parquet" large_df.write_parquet_with_options( - tmp_path, ParquetWriterOptions(max_row_group_size=max_row_group_size) + path, ParquetWriterOptions(max_row_group_size=max_row_group_size) ) - for file in tmp_path.rglob("*.parquet"): - parquet = pq.ParquetFile(file) - metadata = parquet.metadata.to_dict() - for row_group in metadata["row_groups"]: - assert row_group["num_rows"] == max_row_group_size + parquet = pq.ParquetFile(path) + metadata = parquet.metadata.to_dict() + for row_group in metadata["row_groups"]: + assert row_group["num_rows"] == max_row_group_size @pytest.mark.parametrize("created_by", ["datafusion", "datafusion-python", "custom"]) diff --git a/python/tests/test_expr.py b/python/tests/test_expr.py index 40a98dc4..cfeb07c1 100644 --- a/python/tests/test_expr.py +++ b/python/tests/test_expr.py @@ -651,7 +651,7 @@ def test_alias_with_metadata(df): "9033e0e305f247c0c3c80d0c7848c8b3", None, ], - type=pa.string(), + type=pa.string_view(), ), id="md5", ), diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index 5189b651..52591531 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -685,7 +685,8 @@ def test_array_function_obj_tests(stmt, py_expr): "8b1a9953c4611296a827abf8c47804d7", "f5a7924e621e84c9280a9a27e1bcb7f6", "9033e0e305f247c0c3c80d0c7848c8b3", - ] + ], + type=pa.string_view(), ), ), (f.octet_length(column("a")), pa.array([5, 5, 1], type=pa.int32())), diff --git a/src/expr.rs b/src/expr.rs index f1e00236..7e7505dc 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion::logical_expr::expr::AggregateFunctionParams; +use datafusion::logical_expr::expr::{AggregateFunctionParams, FieldMetadata}; use datafusion::logical_expr::utils::exprlist_to_fields; use datafusion::logical_expr::{ lit_with_metadata, ExprFuncBuilder, ExprFunctionExt, LogicalPlan, WindowFunctionDefinition, @@ -283,7 +283,8 @@ impl PyExpr { value: PyScalarValue, metadata: HashMap, ) -> PyExpr { - lit_with_metadata(value.0, metadata).into() + let metadata = FieldMetadata::new(metadata.into_iter().collect()); + lit_with_metadata(value.0, Some(metadata)).into() } #[staticmethod] @@ -294,6 +295,7 @@ impl PyExpr { /// assign a name to the PyExpr #[pyo3(signature = (name, metadata=None))] pub fn alias(&self, name: &str, metadata: Option>) -> PyExpr { + let metadata = metadata.map(|m| FieldMetadata::new(m.into_iter().collect())); self.expr.clone().alias_with_metadata(name, metadata).into() } diff --git a/src/expr/copy_to.rs b/src/expr/copy_to.rs index 473dabfe..c2f7c61d 100644 --- a/src/expr/copy_to.rs +++ b/src/expr/copy_to.rs @@ -73,13 +73,13 @@ impl PyCopyTo { options: HashMap, ) -> Self { PyCopyTo { - copy: CopyTo { - input: input.plan(), + copy: CopyTo::new( + input.plan(), output_url, partition_by, - file_type: file_type.file_type, + file_type.file_type, options, - }, + ), } } diff --git a/src/expr/join.rs b/src/expr/join.rs index b8d1d9da..7b7e0d9d 100644 --- a/src/expr/join.rs +++ b/src/expr/join.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use datafusion::common::NullEquality; use datafusion::logical_expr::logical_plan::{Join, JoinConstraint, JoinType}; use pyo3::{prelude::*, IntoPyObjectExt}; use std::fmt::{self, Display, Formatter}; @@ -116,7 +117,7 @@ impl Display for PyJoin { JoinType: {:?} JoinConstraint: {:?} Schema: {:?} - NullEqualsNull: {:?}", + NullEquality: {:?}", &self.join.left, &self.join.right, &self.join.on, @@ -124,7 +125,7 @@ impl Display for PyJoin { &self.join.join_type, &self.join.join_constraint, &self.join.schema, - &self.join.null_equals_null, + &self.join.null_equality, ) } } @@ -173,7 +174,10 @@ impl PyJoin { /// If null_equals_null is true, null == null else null != null fn null_equals_null(&self) -> PyResult { - Ok(self.join.null_equals_null) + match self.join.null_equality { + NullEquality::NullEqualsNothing => Ok(false), + NullEquality::NullEqualsNull => Ok(true), + } } fn __repr__(&self) -> PyResult { diff --git a/src/expr/literal.rs b/src/expr/literal.rs index 45303a10..561242c9 100644 --- a/src/expr/literal.rs +++ b/src/expr/literal.rs @@ -16,22 +16,18 @@ // under the License. use crate::errors::PyDataFusionError; -use datafusion::common::ScalarValue; +use datafusion::{common::ScalarValue, logical_expr::expr::FieldMetadata}; use pyo3::{prelude::*, IntoPyObjectExt}; -use std::collections::BTreeMap; #[pyclass(name = "Literal", module = "datafusion.expr", subclass)] #[derive(Clone)] pub struct PyLiteral { pub value: ScalarValue, - pub metadata: Option>, + pub metadata: Option, } impl PyLiteral { - pub fn new_with_metadata( - value: ScalarValue, - metadata: Option>, - ) -> PyLiteral { + pub fn new_with_metadata(value: ScalarValue, metadata: Option) -> PyLiteral { Self { value, metadata } } } diff --git a/src/functions.rs b/src/functions.rs index eeef4838..366c59a1 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -19,6 +19,7 @@ use std::collections::HashMap; use datafusion::functions_aggregate::all_default_aggregate_functions; use datafusion::functions_window::all_default_window_functions; +use datafusion::logical_expr::expr::FieldMetadata; use datafusion::logical_expr::expr::WindowFunctionParams; use datafusion::logical_expr::ExprFunctionExt; use datafusion::logical_expr::WindowFrame; @@ -210,6 +211,7 @@ fn order_by(expr: PyExpr, asc: bool, nulls_first: bool) -> PyResult #[pyo3(signature = (expr, name, metadata=None))] fn alias(expr: PyExpr, name: &str, metadata: Option>) -> PyResult { let relation: Option = None; + let metadata = metadata.map(|m| FieldMetadata::new(m.into_iter().collect())); Ok(PyExpr { expr: datafusion::logical_expr::Expr::Alias( Alias::new(expr.expr, relation, name).with_metadata(metadata), @@ -694,7 +696,7 @@ pub fn last_value( null_treatment: Option, ) -> PyDataFusionResult { // If we initialize the UDAF with order_by directly, then it gets over-written by the builder - let agg_fn = functions_aggregate::expr_fn::last_value(expr.expr, None); + let agg_fn = functions_aggregate::expr_fn::last_value(expr.expr, vec![]); add_builder_fns_to_aggregate(agg_fn, distinct, filter, order_by, null_treatment) } @@ -710,7 +712,7 @@ pub fn first_value( null_treatment: Option, ) -> PyDataFusionResult { // If we initialize the UDAF with order_by directly, then it gets over-written by the builder - let agg_fn = functions_aggregate::expr_fn::first_value(expr.expr, None); + let agg_fn = functions_aggregate::expr_fn::first_value(expr.expr, vec![]); add_builder_fns_to_aggregate(agg_fn, distinct, filter, order_by, null_treatment) } From d660fb85529d5bec05cdaa19e145894c6d5ceddf Mon Sep 17 00:00:00 2001 From: nuno-faria Date: Tue, 19 Aug 2025 12:58:21 +0100 Subject: [PATCH 2/3] Bump datafusion to 49.0.1 --- Cargo.lock | 134 ++++++++++----------- Cargo.toml | 10 +- examples/datafusion-ffi-example/Cargo.toml | 4 +- 3 files changed, 74 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 67986df9..1fccb498 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -888,9 +888,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f47772c28553d837e12cdcc0fb04c2a0fe8eca8b704a30f721d076f32407435" +checksum = "f4539076347adb068dd1950f64c6437e946dd3449b8284c971bd05f025a2e648" dependencies = [ "arrow", "arrow-ipc", @@ -944,9 +944,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b6b29c9c922959285fac53139e12c81014e2ca54704f20355edd7e9d11fd773" +checksum = "21ae252e9dd4b0ee0e505fecc94dc74d08eb71dbb47bd252bc27dfeac4ab1fd4" dependencies = [ "arrow", "async-trait", @@ -970,9 +970,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7313553e4c01d184dd49183afdfa22f23204a10a26dd12e6f799203d8fdb95c2" +checksum = "f836ede0cbe4555b47e324fc185a1d5e7816fdc73e40c7669e257834fc488afa" dependencies = [ "arrow", "async-trait", @@ -993,9 +993,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d66104731b7476a8c86fbe7a6fd741e6329791166ac89a91fcd8336a560ddaf" +checksum = "39569c76b0e43ab487fa7ab8d43f9609c013d8da381b093b67e7d12de818271b" dependencies = [ "ahash", "apache-avro", @@ -1020,9 +1020,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7527ecdfeae6961a8564d3b036507a67bd467fd36a9f10cf8ad7a99db1f1bc" +checksum = "68742dd360aaa0746b54f834b9621f3836967359158b1fa13d97b21b8a1defa0" dependencies = [ "futures", "log", @@ -1031,9 +1031,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40e5076be33d8eb9f4d99858e5f3477b36c07e61eee8eb93c4320428d9e1e344" +checksum = "1ec6180c4ddfee5bc384c50ca066e5363a4da7e5d6acf5ac2ac1ddaea5d2cdcb" dependencies = [ "arrow", "async-compression", @@ -1067,9 +1067,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "831cfe556658133ea4270d616164ce27f737e9e4d5e359e1b1b269e0bf767cef" +checksum = "cc9237b6eab947eb0e2bd52e67a8f8fcf084f99352b01f6cb48d4e9d1b33848b" dependencies = [ "apache-avro", "arrow", @@ -1092,9 +1092,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "785518d0f2f136c19b9389a10762c01a5aeb5fcdebdb244297bb656b2862dc88" +checksum = "7c71c8e924db8eae39fd9c1203a8836bd9f2232831e2d07fd8428afe9ba314a2" dependencies = [ "arrow", "async-trait", @@ -1117,9 +1117,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71cb7c3bad0951bf5c52505d0e6d87e6c0098156d2a195924cbcdc82238d29ba" +checksum = "fcb282c51182b741e3cb583fcf419f75928bb89742d1b3ac6c1ad9b188ef1d6d" dependencies = [ "arrow", "async-trait", @@ -1142,9 +1142,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea76ad2c5189c98a6b1d4bdf6c3b3caacc9701c417af6661597c946a201bc328" +checksum = "822d18a53a86a5354716828116dd541389838e73ff069149a6f09bd14bcfd2fa" dependencies = [ "arrow", "async-trait", @@ -1175,15 +1175,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bcc45e380db5c6033c3f39e765a3d752679f14315060a7f4030a60066a36946" +checksum = "f5d50e710c1a4b944fa75e5a184eafe4a70bc1016413f0388949e8be3383f218" [[package]] name = "datafusion-execution" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8209805fdce3d5c6e1625f674d3e4ce93e995a56d3709a0bb8d4361062652596" +checksum = "ed30edd30710767642f5d78e2c568354a51c342713874305d8281bcf64895a7d" dependencies = [ "arrow", "dashmap", @@ -1200,9 +1200,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7879a845e72a00cacffacbdf5f40626049cb9584d2ba8aa0b9172f09833110ab" +checksum = "50af1197648e4c860414d1871aa73f1cadf547ed65dc7a88720e961f9def5a9f" dependencies = [ "arrow", "async-trait", @@ -1222,9 +1222,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6da7e47e70ef2c7678735c82c392bd74687004043f5fc8072ab8678dc6fa459d" +checksum = "f492ba3c522d103690c84c3d38039d83d4b2cf26aab7f0038f77e47298aad5ef" dependencies = [ "arrow", "datafusion-common", @@ -1235,9 +1235,9 @@ dependencies = [ [[package]] name = "datafusion-ffi" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba959e6fd3a9a503201600e5575b3cb2aaa066b876317218ce25f0c0f836ac5f" +checksum = "79d43210e2e9c006b1be88d73fee467858382bc1928b827fd199518d1a226c3a" dependencies = [ "abi_stable", "arrow", @@ -1257,9 +1257,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e7b92b04c5c3b1151f055251b36e272071f9088d9701826a533cb4f764af1c8" +checksum = "55ef6ad5fde3f35e5da5848653936e6a75f565e8551a24da952cb26036f9babd" dependencies = [ "arrow", "arrow-buffer", @@ -1286,9 +1286,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f16cb922b62e535a4d484961ac2c1c6d188dbe02e85e026c05f0fabbc8f814e" +checksum = "e5fa9ca2f7097b7716094de7a9f70635cd56881476dd274a1f96f30f199efaed" dependencies = [ "ahash", "arrow", @@ -1307,9 +1307,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f71bb59dc8b4dc985c911f2e0d8cf426c21f565b56dca4b852c244101a1a7a2" +checksum = "14c782fca3f8fcd82bbef57f39a6581c807dae961e85853f413161775d77ab39" dependencies = [ "ahash", "arrow", @@ -1320,9 +1320,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27eb3b98a2eb02a8af4ef19cc793cac21fc98d8720b987f15d7d25b8cc875f4d" +checksum = "7a85e8e1856a93aa4ae7c0c909d12c9384809739d53556023ae184e708f51f4f" dependencies = [ "arrow", "arrow-ord", @@ -1342,9 +1342,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350e0940fc3e2fa4645a4d323f9ebf9258b2d7fdad12013a471cae4ae5568683" +checksum = "5325b42aa4f775f3789310b8760681027460c2407579f2d213bca03e29ecf75b" dependencies = [ "arrow", "async-trait", @@ -1358,9 +1358,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df03c6c62039578fd110b327c474846fdf3d9077a568f1e8706e585ed30cb98d" +checksum = "fa2129e59f63dc15ea827da07d291093361df380135b86589d946b4ef76547ba" dependencies = [ "arrow", "datafusion-common", @@ -1376,9 +1376,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "083659a95914bf3ca568a72b085cb8654576fef1236b260dc2379cb8e5f922b2" +checksum = "8b60f56521ffc46e093eceec9eec4da2ae604d37aaf15dbcb6c812403173b49f" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1386,9 +1386,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cabe1f32daa2fa54e6b20d14a13a9e85bef97c4161fe8a90d76b6d9693a5ac4" +checksum = "553cd882a5a8e3ecb1b1af1b33bbb898ddd3459020e2eff87fb177146447dee4" dependencies = [ "datafusion-expr", "quote", @@ -1397,9 +1397,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e12a97dcb0ccc569798be1289c744829cce5f18cc9b037054f8d7f93e1d57be" +checksum = "fbe958375ac1bb4a81749dda9712d6a7572b3d3f0bec4e23134fa68d6546b5b5" dependencies = [ "arrow", "chrono", @@ -1417,9 +1417,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41312712b8659a82b4e9faa8d97a018e7f2ccbdedf2f7cb93ecf256e39858c86" +checksum = "b50077fd5b584d7f3bf3a9687e34213c36945300e8348d0bb14d2a5a8af38ffd" dependencies = [ "ahash", "arrow", @@ -1439,9 +1439,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be1649a60ea0319496d616ae3554e84dfcc262c201ab4439abcd83cca989b85b" +checksum = "acea27a3897b793c1eb54c7ab9b6eea1827a0263e88029ca04a6d08b1e2a4c6a" dependencies = [ "ahash", "arrow", @@ -1453,9 +1453,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea3f5b8ba6122426774aaaf11325740b8e5d3afaab9ab39dc63423adca554748" +checksum = "06da58128f25036526783830c5d7dea85fffa48bbff3f640aa1f90c1e184e803" dependencies = [ "arrow", "datafusion-common", @@ -1473,9 +1473,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a595f296929d6cffa12b993ea53e9fe8215fada050d78626c5cf0e2f02b0205" +checksum = "35085e346acb5348733ed31696a639ef12788bc1cb09abfcebb66f5cd283fba6" dependencies = [ "ahash", "arrow", @@ -1503,9 +1503,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ab6f4fa0f3bbfbc0b4f89485bcd7cbed6cca0347e8d1eda50b66b779725b6e" +checksum = "2d5be95356413f502e8c474335a9e7f345451b7e395a031c10eba51c9929f63c" dependencies = [ "arrow", "chrono", @@ -1519,9 +1519,9 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ba94d76d85459ebbf7c29aa1b001234d551e840192c742a4237ec22de45a557" +checksum = "f74961518bea403a37a8eee85d7963b002184dd18aafa68114df85fea37fe4db" dependencies = [ "arrow", "datafusion-common", @@ -1548,7 +1548,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "async-trait", @@ -1573,9 +1573,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd5f2fe790f43839c70fb9604c4f9b59ad290ef64e1d2f927925dd34a9245406" +checksum = "2b475806da727769daa1e667891c4e3b3e4353b49a3cd492a90c443b71230b19" dependencies = [ "arrow", "async-trait", @@ -1597,9 +1597,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ebebb82fda37f62f06fe14339f4faa9f197a0320cc4d26ce2a5fd53a5ccd27c" +checksum = "5b0834fda1f2834d21b7bedb663c458e1e240e4f9053db86b69003edabdd2baa" dependencies = [ "arrow", "bigdecimal", @@ -1614,9 +1614,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d5a2bb4746c340a59cb9bdb4728826fff71d116a2a8e1c235f0988def3f5fdc" +checksum = "b864df7ccc4b3645a9e25069657650ebb11b9f19058fe0c670ee0684465855c6" dependencies = [ "async-recursion", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 8d8d3a5c..f58ae275 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "49.0.0" +version = "49.0.1" homepage = "https://datafusion.apache.org/python" repository = "https://github.com/apache/datafusion-python" authors = ["Apache DataFusion "] @@ -39,10 +39,10 @@ pyo3 = { version = "0.24", features = ["extension-module", "abi3", "abi3-py39"] pyo3-async-runtimes = { version = "0.24", features = ["tokio-runtime"]} pyo3-log = "0.12.4" arrow = { version = "55.1.0", features = ["pyarrow"] } -datafusion = { version = "49.0.0", features = ["avro", "unicode_expressions"] } -datafusion-substrait = { version = "49.0.0", optional = true } -datafusion-proto = { version = "49.0.0" } -datafusion-ffi = { version = "49.0.0" } +datafusion = { version = "49.0.1", features = ["avro", "unicode_expressions"] } +datafusion-substrait = { version = "49.0.1", optional = true } +datafusion-proto = { version = "49.0.1" } +datafusion-ffi = { version = "49.0.1" } prost = "0.13.1" # keep in line with `datafusion-substrait` uuid = { version = "1.16", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } diff --git a/examples/datafusion-ffi-example/Cargo.toml b/examples/datafusion-ffi-example/Cargo.toml index 65a58858..cd72ffbc 100644 --- a/examples/datafusion-ffi-example/Cargo.toml +++ b/examples/datafusion-ffi-example/Cargo.toml @@ -21,8 +21,8 @@ version = "0.2.0" edition = "2021" [dependencies] -datafusion = { version = "49.0.0" } -datafusion-ffi = { version = "49.0.0" } +datafusion = { version = "49.0.1" } +datafusion-ffi = { version = "49.0.1" } pyo3 = { version = "0.23", features = ["extension-module", "abi3", "abi3-py39"] } arrow = { version = "55.0.0" } arrow-array = { version = "55.0.0" } From f52c9fa2f0935132d47864b84993ff9a4eb3c9e0 Mon Sep 17 00:00:00 2001 From: nuno-faria Date: Tue, 19 Aug 2025 19:11:57 +0100 Subject: [PATCH 3/3] Fix clippy errors --- examples/datafusion-ffi-example/Cargo.lock | 8 ++++---- src/common/schema.rs | 2 +- src/dataframe.rs | 2 +- src/expr.rs | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/datafusion-ffi-example/Cargo.lock b/examples/datafusion-ffi-example/Cargo.lock index e2ebc692..b25bf472 100644 --- a/examples/datafusion-ffi-example/Cargo.lock +++ b/examples/datafusion-ffi-example/Cargo.lock @@ -759,9 +759,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f47772c28553d837e12cdcc0fb04c2a0fe8eca8b704a30f721d076f32407435" +checksum = "f4539076347adb068dd1950f64c6437e946dd3449b8284c971bd05f025a2e648" dependencies = [ "arrow", "arrow-ipc", @@ -1079,9 +1079,9 @@ dependencies = [ [[package]] name = "datafusion-ffi" -version = "49.0.0" +version = "49.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba959e6fd3a9a503201600e5575b3cb2aaa066b876317218ce25f0c0f836ac5f" +checksum = "79d43210e2e9c006b1be88d73fee467858382bc1928b827fd199518d1a226c3a" dependencies = [ "abi_stable", "arrow", diff --git a/src/common/schema.rs b/src/common/schema.rs index 5a54fe33..752c39bd 100644 --- a/src/common/schema.rs +++ b/src/common/schema.rs @@ -198,7 +198,7 @@ impl TableSource for SqlTableSource { .collect() } - fn get_logical_plan(&self) -> Option> { + fn get_logical_plan(&self) -> Option> { None } } diff --git a/src/dataframe.rs b/src/dataframe.rs index ab4749e3..05f665cd 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -152,7 +152,7 @@ fn get_python_formatter_with_config(py: Python) -> PyResult { } /// Get the Python formatter from the datafusion.dataframe_formatter module -fn import_python_formatter(py: Python) -> PyResult> { +fn import_python_formatter(py: Python<'_>) -> PyResult> { let formatter_module = py.import("datafusion.dataframe_formatter")?; let get_formatter = formatter_module.getattr("get_formatter")?; get_formatter.call0() diff --git a/src/expr.rs b/src/expr.rs index 7e7505dc..e2c53025 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -698,7 +698,7 @@ impl PyExpr { /// Create a [Field] representing an [Expr], given an input [LogicalPlan] to resolve against pub fn expr_to_field(expr: &Expr, input_plan: &LogicalPlan) -> PyDataFusionResult> { - let fields = exprlist_to_fields(&[expr.clone()], input_plan)?; + let fields = exprlist_to_fields(std::slice::from_ref(expr), input_plan)?; Ok(fields[0].1.clone()) } fn _types(expr: &Expr) -> PyResult {